def get_client(image=None, quiet=False, **kwargs):

from sregistry.defaults import SREGISTRY_CLIENT<EOL>if not check_install():<EOL><INDENT>bot.warning('<STR_LIT>')<EOL><DEDENT>client_name = get_uri(image)<EOL>if client_name is not None:<EOL><INDENT>SREGISTRY_CLIENT = client_name<EOL><DEDENT>if   SREGISTRY_CLIENT == '<STR_LIT>': from .aws import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .docker import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .dropbox import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .gitlab import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .globus import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .nvidia import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .hub import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .google_drive import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .google_storage import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .google_storage import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .google_build import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .registry import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .s3 import Client<EOL>elif SREGISTRY_CLIENT == '<STR_LIT>': from .swift import Client<EOL>else: from .hub import Client<EOL>Client.client_name = SREGISTRY_CLIENT<EOL>Client.quiet = quiet<EOL>Client._credential_cache = get_credential_cache()<EOL>if SREGISTRY_DATABASE is not None:<EOL><INDENT>from sregistry.database import (<EOL>init_db, add, cp, get, mv, rm, rmi, <EOL>images, inspect,<EOL>rename,<EOL>get_container,<EOL>get_collection, <EOL>get_or_create_collection <EOL>)<EOL>Client._init_db = init_db<EOL>Client.add = add<EOL>Client.cp = cp<EOL>Client.get = get<EOL>Client.inspect = inspect<EOL>Client.mv = mv<EOL>Client.rename = rename<EOL>Client.rm = rm<EOL>Client.rmi = rmi<EOL>Client.images = images<EOL>Client.get_or_create_collection = get_or_create_collection<EOL>Client.get_container = get_container<EOL>Client.get_collection = get_collection<EOL><DEDENT>else:<EOL><INDENT>from sregistry.database import ( add, init_db )<EOL>Client.add = add<EOL>Client._init_db = init_db        <EOL><DEDENT>cli = Client()<EOL>if hasattr(Client, '<STR_LIT>'):<EOL><INDENT>cli._init_db(SREGISTRY_DATABASE)<EOL><DEDENT>return cli<EOL>

get the correct client depending on the driver of interest. The
selected client can be chosen based on the environment variable
SREGISTRY_CLIENT, and later changed based on the image uri parsed
If there is no preference, the default is to load the singularity
hub client.

Parameters
==========
image: if provided, we derive the correct client based on the uri
of an image. If not provided, we default to environment, then hub.
quiet: if True, suppress most output about the client (e.g. speak)


def pull(self, images, file_name=None, save=True, force=False, **kwargs):

if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' % len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name(remove_uri(image), lowercase=False)<EOL>url = "<STR_LIT>" %(self.base, q['<STR_LIT>'], <EOL>q['<STR_LIT:image>'],<EOL>q['<STR_LIT>'])<EOL>bot.debug('<STR_LIT>' % url)<EOL>manifest = self._get(url)<EOL>manifest['<STR_LIT>'] = url        <EOL>if "<STR_LIT:version>" in manifest:<EOL><INDENT>q = parse_image_name('<STR_LIT>' %(q['<STR_LIT>'], manifest['<STR_LIT:version>']))<EOL><DEDENT>if file_name is None:<EOL><INDENT>file_name = self._get_storage_name(q)<EOL><DEDENT>file_name = os.path.abspath(file_name)<EOL>if os.path.exists(file_name) and force is False:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>show_bar = not bool(self.quiet)<EOL>image_file = self.download(url=manifest['<STR_LIT:image>'],<EOL>file_name=os.path.basename(file_name),<EOL>show_progress=show_bar)<EOL>if save is True:<EOL><INDENT>image_uri = "<STR_LIT>" %(manifest['<STR_LIT:name>'], manifest['<STR_LIT>'], manifest['<STR_LIT:version>'])<EOL>container = self.add(image_path=image_file,<EOL>image_uri=image_uri,<EOL>image_name=file_name,<EOL>metadata=manifest,<EOL>url=manifest['<STR_LIT:image>'])<EOL>image_file = container.image<EOL><DEDENT>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' %image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL>finished.append(image_file)<EOL><DEDENT>file_name = None<EOL><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from an endpoint

        Parameters
        ==========
        images: refers to the uri given by the user to pull in the format
        <collection>/<namespace>. You should have an API that is able to
        retrieve a container based on parsing this uri.
        file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
        save: if True, you should save the container to the database
          using self.add()

        Returns
        =======
        finished: a single container path, or list of paths


if query is not None:<EOL><INDENT>return self._search_collection(query)<EOL><DEDENT>return self.list()<EOL>

query a Singularity registry for a list of images.
     If query is None, collections are listed.

    EXAMPLE QUERIES:

    [empty]             list all collections in singularity hub
    vsoch               do a general search for collection "vsoch"
    vsoch/dinosaur      list details of container vsoch/dinosaur
                          tag "latest" is used by default, and then the most recent
    vsoch/dinosaur:tag  list details for specific container


quiet=False<EOL>if "<STR_LIT>" in kwargs:<EOL><INDENT>quiet = kwargs['<STR_LIT>']<EOL><DEDENT>bot.spinner.start()<EOL>url = '<STR_LIT>' %self.base<EOL>results = self._paginate_get(url)<EOL>bot.spinner.stop()   <EOL>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>rows = []<EOL>for result in results:<EOL><INDENT>if "<STR_LIT>" in result:<EOL><INDENT>if result['<STR_LIT:id>'] not in [<NUM_LIT>,<NUM_LIT>,<NUM_LIT>]:<EOL><INDENT>for c in result['<STR_LIT>']:<EOL><INDENT>rows.append([c['<STR_LIT>'],"<STR_LIT>" %(c['<STR_LIT:name>'],c['<STR_LIT>'])])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if quiet is False:<EOL><INDENT>bot.info("<STR_LIT>")<EOL>bot.table(rows)<EOL><DEDENT>return rows<EOL>

a "show all" search that doesn't require a query


query = query.lower().strip('<STR_LIT:/>')<EOL>q = parse_image_name(remove_uri(query), defaults=False)<EOL>containers = self.list(quiet=True)<EOL>rows = []<EOL>for result in containers:<EOL><INDENT>if re.search(query, result[<NUM_LIT:1>]):<EOL><INDENT>rows.append(result)<EOL><DEDENT><DEDENT>if len(rows) > <NUM_LIT:0>:<EOL><INDENT>bot.table(rows)<EOL><DEDENT>else:<EOL><INDENT>bot.info('<STR_LIT>')    <EOL><DEDENT>return rows<EOL>

collection search will list all containers for a specific
    collection. We assume query is the name of a collection


if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name( remove_uri(image), <EOL>default_collection='<STR_LIT>' )<EOL>image_file = self._pull(file_name=file_name, <EOL>uri='<STR_LIT>',<EOL>save=save, <EOL>force=force, <EOL>names=q,<EOL>kwargs=kwargs)<EOL>finished.append(image_file)<EOL><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from a docker hub. This is a (less than ideal) workaround
       that actually does the following:

       - creates a sandbox folder
       - adds docker layers, metadata folder, and custom metadata to it
       - converts to a squashfs image with build

    the docker manifests are stored with registry metadata.

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths


def __init__(self, secrets=None, **kwargs):

self._set_base()<EOL>self.reverseLayers = False<EOL>self._reset_headers()<EOL>self._update_secrets()<EOL>self._update_headers()<EOL>super(ApiConnection, self).__init__(**kwargs)<EOL>

to work with nvidia container registry, we do the following:

        1. set the base to index.docker.io, or defined in environment
        2. assume starting with v2 schema, we won't reverse layers
        3. set headers to ask for version 2 schema first
        4. update secrets, 1st priority environment, then .docker/config.json
        5. update headers based on secrets


pass<EOL>

if you want to add an extra print (of a parameter, for example)
           for the user when the client initalizes, write it here, eg:
           bot.info('[setting] value')


<EOL>accept = '<STR_LIT>'<EOL>accept += '<STR_LIT>'<EOL>accept += '<STR_LIT:application/json>'<EOL>self.headers = {"<STR_LIT>": accept,<EOL>'<STR_LIT:Content-Type>': '<STR_LIT>'}<EOL>

reset headers is called from update_headers, and will update the
           headers based on what is found with the client secrets


base = self._get_setting('<STR_LIT>')<EOL>version = self._get_setting('<STR_LIT>')<EOL>if base is None:<EOL><INDENT>base = "<STR_LIT>"<EOL><DEDENT>if version is None:<EOL><INDENT>version = "<STR_LIT>"<EOL><DEDENT>nohttps = self._get_setting('<STR_LIT>')<EOL>if nohttps is None:<EOL><INDENT>nohttps = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>nohttps = "<STR_LIT>"<EOL><DEDENT>self.base = "<STR_LIT>" %(nohttps, base.strip('<STR_LIT:/>'), version)<EOL>

set the API base or default to use Docker Hub. The user is able
           to set the base, api version, and protocol via a settings file
           of environment variables:

           SREGISTRY_NVIDIA_BASE: defaults to nvcr.io
           SREGISTRY_NVIDIA_TOKEN: defaults to $oauthtoken
           SREGISTRY_NVIDIA_VERSION: defaults to v2
           SREGISTRY_NVIDIA_NO_HTTPS: defaults to not set (so https)


<EOL>token = self._required_get_and_update('<STR_LIT>')<EOL>username = self._get_and_update_setting('<STR_LIT>')<EOL>if username is None:<EOL><INDENT>username = "<STR_LIT>"<EOL><DEDENT>if token is not None:<EOL><INDENT>auth = basic_auth_header(username, token)<EOL>self.headers.update(auth)<EOL><DEDENT>

update secrets will take a secrets credential file
           either located at .sregistry or the environment variable
           SREGISTRY_CLIENT_SECRETS and update the current client
           secrets as well as the associated API base. For the case of
           using Docker Hub, if we find a .docker secrets file, we update
           from there.


path = os.path.abspath(path)<EOL>bot.debug("<STR_LIT>" % path)<EOL>if not os.path.exists(path):<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name(remove_uri(name),tag=tag)<EOL>metadata = self.get_metadata(path, names=names)<EOL>bot.spinner.start()<EOL>bot.spinner.stop()<EOL>

push an image to Singularity Registry

    path: should correspond to an absolte image path (or derive it)
    name: should be the complete uri that the user has requested to push.
    tag: should correspond with an image tag. This is provided to mirror Docker


def pull(self, images, file_name=None, save=True, **kwargs):

<EOL>if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name(remove_uri(image))<EOL>url = "<STR_LIT>" <EOL>bot.debug('<STR_LIT>' %url)<EOL>manifest = self._get(url)<EOL>manifest['<STR_LIT>'] = url<EOL>if file_name is None:<EOL><INDENT>file_name = q['<STR_LIT>'].replace('<STR_LIT:/>','<STR_LIT:->')<EOL><DEDENT>image_file = self.download(url=manifest['<STR_LIT:image>'],<EOL>file_name=file_name,<EOL>show_progress=True)<EOL>if save is True:<EOL><INDENT>image_uri = "<STR_LIT>" %(manifest['<STR_LIT>'], <EOL>manifest['<STR_LIT:name>'],<EOL>manifest['<STR_LIT>'])<EOL>container = self.add(image_path = image_file, <EOL>image_uri = image_uri,   <EOL>image_name = file_name, <EOL>metadata = manifest,<EOL>url = manifest['<STR_LIT:image>'])<EOL>image_file = container.image<EOL><DEDENT>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' %image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL>finished.append(image_file)<EOL><DEDENT><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from a singularity registry

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths


def search(self, query=None, args=None):

<EOL>if query is not None:<EOL><INDENT>return self._container_query(query)<EOL><DEDENT>return self._search_all()<EOL>

query a Singularity registry for a list of images.
     If query is None, collections are listed.

    EXAMPLE QUERIES:


<EOL>url = '<STR_LIT>'<EOL>results = self._paginate_get(url)<EOL>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>bot.info("<STR_LIT>")<EOL>rows = []<EOL>for result in results:<EOL><INDENT>if "<STR_LIT>" in result:<EOL><INDENT>for c in result['<STR_LIT>']:<EOL><INDENT>rows.append([ c['<STR_LIT>'],<EOL>c['<STR_LIT>'] ])<EOL><DEDENT><DEDENT><DEDENT>bot.table(rows)<EOL>return rows<EOL>


<EOL>rows = ['<STR_LIT>','<STR_LIT>','<STR_LIT>']<EOL>bot.table(rows)<EOL>return rows<EOL>

search for a specific container.
    This function would likely be similar to the above, but have different
    filter criteria from the user (based on the query)


pass<EOL>


<EOL>setting = self._get_setting('<STR_LIT>')<EOL>setting = self._get_and_update_setting('<STR_LIT>')<EOL>self.secrets = read_client_secrets()<EOL>if self._credential_cache is not None:<EOL><INDENT>bot.info("<STR_LIT>" %self._credential_cache)<EOL><DEDENT>

update secrets will take a secrets credential file
           either located at .sregistry or the environment variable
           SREGISTRY_CLIENT_SECRETS and update the current client
           secrets as well as the associated API base. This is where you
           should do any customization of the secrets flie, or using
           it to update your client, if needed.


<EOL>parent = self._get_or_create_folder(self._base)<EOL>image = None<EOL>path = os.path.abspath(path)<EOL>bot.debug("<STR_LIT>" % path)<EOL>if not os.path.exists(path):<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name(remove_uri(name),tag=tag)<EOL>if names['<STR_LIT:version>'] is None:<EOL><INDENT>version = get_image_hash(path)<EOL>names = parse_image_name(remove_uri(name), tag=tag, version=version)<EOL><DEDENT>metadata = self.get_metadata(path, names=names)<EOL>metadata = metadata['<STR_LIT:data>']<EOL>metadata.update(names)<EOL>metadata.update(metadata['<STR_LIT>']['<STR_LIT>'])<EOL>del metadata['<STR_LIT>']<EOL>file_metadata = {<EOL>'<STR_LIT:name>': names['<STR_LIT>'],<EOL>'<STR_LIT>' : '<STR_LIT>',<EOL>'<STR_LIT>': [parent['<STR_LIT:id>']],<EOL>'<STR_LIT>': metadata<EOL>}<EOL>media = MediaFileUpload(path,resumable=True)<EOL>try:<EOL><INDENT>bot.spinner.start()<EOL>image = self._service.files().create(body=file_metadata,<EOL>media_body=media,<EOL>fields='<STR_LIT:id>').execute()<EOL>thumbnail = get_thumbnail()<EOL>with open(thumbnail, "<STR_LIT:rb>") as f:<EOL><INDENT>body = { "<STR_LIT>": { <EOL>"<STR_LIT>": { "<STR_LIT:image>": base64.urlsafe_b64encode(f.read()).decode('<STR_LIT:utf8>'),<EOL>"<STR_LIT>": "<STR_LIT>" }<EOL>}}<EOL>image = self._service.files().update(fileId=image['<STR_LIT:id>'],<EOL>body = body).execute()<EOL><DEDENT>bot.spinner.stop()<EOL>print(image['<STR_LIT:name>'])<EOL><DEDENT>except HttpError:<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>pass<EOL><DEDENT>return image<EOL>

push an image to Google Cloud Drive, meaning uploading it

    path: should correspond to an absolte image path (or derive it)
    name: should be the complete uri that the user has requested to push.
    tag: should correspond with an image tag. This is provided to mirror Docker


if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name( remove_uri(image) )<EOL>bot.info('<STR_LIT>' %(q['<STR_LIT>'],self._base))<EOL>matches = self._container_query(q['<STR_LIT>'], quiet=True)<EOL>if len(matches) == <NUM_LIT:0>:<EOL><INDENT>bot.info('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>if file_name is None:<EOL><INDENT>file_name = q['<STR_LIT>'].replace('<STR_LIT:/>','<STR_LIT:->')<EOL><DEDENT>image = matches[<NUM_LIT:0>]<EOL>request = self._service.files().get_media(fileId=image['<STR_LIT:id>'])<EOL>with open(file_name, '<STR_LIT:wb>') as fh:<EOL><INDENT>downloader = MediaIoBaseDownload(fh, request)<EOL>done = False<EOL>bar = None<EOL>while done is False:<EOL><INDENT>status, done = downloader.next_chunk()<EOL>response = None<EOL>if bar is None:<EOL><INDENT>total = status.total_size / (<NUM_LIT>*<NUM_LIT>)<EOL>bar = ProgressBar(expected_size=total,<EOL>filled_char='<STR_LIT:=>',<EOL>hide=self.quiet)<EOL><DEDENT>bar.show(status.resumable_progress / (<NUM_LIT>*<NUM_LIT>))<EOL><DEDENT><DEDENT>if save is True:<EOL><INDENT>image_uri = q['<STR_LIT>']<EOL>if "<STR_LIT>" in image:<EOL><INDENT>image_uri = image['<STR_LIT>']<EOL><DEDENT>image['<STR_LIT>'] = downloader._uri<EOL>container = self.add(image_path = image_file,<EOL>image_uri = image_uri,<EOL>metadata = image,<EOL>url=downloader._uri)<EOL>image_file = container.image<EOL><DEDENT>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' %image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL>finished.append(image_file)<EOL><DEDENT><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from google drive, based on a query (uri or id)

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths


if query is not None:<EOL><INDENT>return self._container_query(query)<EOL><DEDENT>return self._search_all()<EOL>

query a bucket for images that match a particular pattern. If no
       query is provided, all images in the bucket are listed that have type
       "container" in the metadata and client "sregistry"


<EOL>folder = self._get_or_create_folder(self._base)<EOL>next_page = None<EOL>containers = []<EOL>while True:<EOL><INDENT>query = "<STR_LIT>"  <EOL>query += "<STR_LIT>"<EOL>query += "<STR_LIT>" %folder['<STR_LIT:id>']   <EOL>response = self._service.files().list(q=query,<EOL>spaces='<STR_LIT>',<EOL>fields='<STR_LIT>',<EOL>pageToken=next_page).execute()<EOL>containers += response.get('<STR_LIT>', [])<EOL>next_page = response.get('<STR_LIT>')<EOL>if not next_page:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if len(containers) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return containers<EOL>

return a list of containers. Since Google Drive definitely has other
       kinds of files, we look for containers in a special sregistry folder,
       (meaning the parent folder is sregistry) and with properties of type
       as container.


results = self._list_containers()<EOL>matches = []<EOL>bot.info("<STR_LIT>" %self._base)<EOL>rows = []<EOL>for i in results:<EOL><INDENT>uri = i['<STR_LIT:name>'].replace('<STR_LIT>','<STR_LIT>')<EOL>if '<STR_LIT>' in i:<EOL><INDENT>if '<STR_LIT>' in i['<STR_LIT>']:<EOL><INDENT>uri = i['<STR_LIT>']['<STR_LIT>']<EOL><DEDENT><DEDENT>rows.append([i['<STR_LIT:id>'],uri])<EOL>i['<STR_LIT>'] = uri<EOL>matches.append(i)<EOL><DEDENT>bot.custom(prefix="<STR_LIT>" %self._base, <EOL>message="<STR_LIT>", <EOL>color="<STR_LIT>")<EOL>bot.table(rows)<EOL>return matches<EOL>

a "list all" search that doesn't require a query. Here we return to
       the user all objects that have custom properties value type set to
       container, which is set when the image is pushed.

       IMPORTANT: the upload function adds this metadata. For a container to
       be found by the client, it must have the properties value with type
       as container. It also should have a "uri" in properties to show the
       user, otherwise the user will have to query / download based on the id


def container_query(self, query, quiet=False):

results = self._list_containers()<EOL>matches = []<EOL>for result in results:<EOL><INDENT>is_match = False<EOL>if query in result['<STR_LIT:id>']:<EOL><INDENT>is_match = True<EOL><DEDENT>elif query in result['<STR_LIT:name>']:<EOL><INDENT>is_match = True<EOL><DEDENT>else:<EOL><INDENT>for key,val in result['<STR_LIT>'].items():<EOL><INDENT>if query in val and is_match is False:<EOL><INDENT>is_match = True<EOL>break<EOL><DEDENT><DEDENT><DEDENT>if is_match is True:<EOL><INDENT>matches.append(result)<EOL><DEDENT><DEDENT>if not quiet:<EOL><INDENT>bot.info("<STR_LIT>" %(self._base,len(matches)))<EOL>for image in matches:<EOL><INDENT>if '<STR_LIT>' in image:<EOL><INDENT>image.update(image['<STR_LIT>'])<EOL><DEDENT>bot.info(image['<STR_LIT>'])<EOL>for key in sorted(image, key=len):<EOL><INDENT>val = image[key]<EOL>if isinstance(val,str):<EOL><INDENT>bot.custom(prefix=key.ljust(<NUM_LIT:10>), message=val, color="<STR_LIT>")<EOL><DEDENT><DEDENT>bot.newline()<EOL><DEDENT><DEDENT>return matches<EOL>

search for a specific container.
    This function is the same as the search all, but instead of showing all
    results, filters them down based on user criteria (the query)


bot.info('<STR_LIT>' %self._base)<EOL>

add the bucket name to be printed to the user at appropriate times


env = '<STR_LIT>'<EOL>self._secrets = self._get_and_update_setting(env)<EOL>self._base = self._get_and_update_setting('<STR_LIT>')<EOL>if self._base is None:<EOL><INDENT>self._base = '<STR_LIT>'<EOL><DEDENT>if self._secrets is None:<EOL><INDENT>bot.error('<STR_LIT>' %env)<EOL>bot.info("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

The user is required to have an application secrets file in his
           or her environment. The client exists with error
           if the variable isn't found.


self._scope = '<STR_LIT>'<EOL>self._service = self._get_service()<EOL>

init client will check if the user has defined a bucket that
           differs from the default, use the application credentials to
           get the bucket, and then instantiate the client.


invalid = True<EOL>if self._credential_cache is not None:<EOL><INDENT>storage = Storage(self._credential_cache)<EOL>if os.path.exists(self._credential_cache):<EOL><INDENT>credentials = storage.get()<EOL>if not credentials.invalid:<EOL><INDENT>invalid = False<EOL><DEDENT><DEDENT><DEDENT>if invalid is True:<EOL><INDENT>class flags:<EOL><INDENT>auth_host_name='<STR_LIT:localhost>'<EOL>auth_host_port=[<NUM_LIT>]<EOL>noauth_local_webserver=False<EOL>logging_level='<STR_LIT>'<EOL><DEDENT>flow = oclient.flow_from_clientsecrets(self._secrets, self._scope)<EOL>credentials = tools.run_flow(flow, storage, flags)<EOL>if self._credential_cache is not None:<EOL><INDENT>storage.put(credentials)<EOL><DEDENT><DEDENT>http = credentials.authorize(httplib2.Http())<EOL>return build('<STR_LIT>', version, http=http)<EOL>

get service client for the google drive API
        :param version: version to use (default is v3)


q = "<STR_LIT>" %folder<EOL>response = self._service.files().list(q=q,<EOL>spaces='<STR_LIT>').execute().get('<STR_LIT>',[])<EOL>if len(response) == <NUM_LIT:0>:            <EOL><INDENT>folder = self._create_folder(folder)<EOL><DEDENT>else:<EOL><INDENT>folder = response[<NUM_LIT:0>]<EOL><DEDENT>return folder<EOL>

create a folder at the drive root. If the folder already exists,
       it is simply returned.

           folder = self._get_or_create_folder(self._base)
           $ folder
             {'id': '1pXR5S8wufELh9Q-jDkhCoYu-BL1NqN9y'}


folder_metadata = {<EOL>'<STR_LIT:name>': os.path.basename(folder),<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>created = self._service.files().create(body=folder_metadata,<EOL>fields='<STR_LIT:id>').execute()<EOL>return created<EOL>

create a folder with a particular name. Be careful, if the folder
       already exists you can still create one (a different one) with
       the equivalent name!


images = self._container_query(query, quiet=True)<EOL>if len(images) == <NUM_LIT:0>:<EOL><INDENT>bot.error('<STR_LIT>' %query)<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>image = images[<NUM_LIT:0>]<EOL>def callback(request_id, response, exception):<EOL><INDENT>if exception:<EOL><INDENT>print(exception)<EOL><DEDENT>else:<EOL><INDENT>share_id = response.get('<STR_LIT:id>')<EOL>bot.info('<STR_LIT>' %(share_to, share_id))<EOL><DEDENT><DEDENT>batch = self._service.new_batch_http_request(callback=callback)<EOL>user_permission = {<EOL>'<STR_LIT:type>': '<STR_LIT:user>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': share_to<EOL>}<EOL>batch.add(self._service.permissions().create(<EOL>fileId=image['<STR_LIT:id>'],<EOL>body=user_permission,<EOL>fields='<STR_LIT:id>',<EOL>))<EOL>batch.execute()<EOL>return image<EOL>

share will use the client to get an image based on a query, and then
       the link with an email or endpoint (share_to) of choice.


<EOL>endpoint, remote = self._parse_endpoint_name(name)<EOL>path = os.path.abspath(path)<EOL>image = os.path.basename(path)<EOL>bot.debug("<STR_LIT>" % path)<EOL>q = parse_image_name(image)<EOL>if not os.path.exists(path):<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._init_transfer_client()<EOL><DEDENT>endpoints = self._get_endpoints()<EOL>if len(endpoints['<STR_LIT>']) == <NUM_LIT:0>:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>) <EOL><DEDENT>source_endpoint = None<EOL>for eid,contender in endpoints['<STR_LIT>'].items():<EOL><INDENT>if contender['<STR_LIT>'] is True:<EOL><INDENT>source_endpoint = contender<EOL>break<EOL><DEDENT><DEDENT>if source_endpoint is None:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>self._create_endpoint_cache(endpoint)<EOL>added = self.add(image_path=path, <EOL>image_uri=q['<STR_LIT>'],<EOL>copy=True)<EOL>label = "<STR_LIT>" %added.name<EOL>tdata = globus_sdk.TransferData(self.transfer_client, <EOL>source_endpoint['<STR_LIT:id>'],<EOL>endpoint,<EOL>label=label,<EOL>sync_level="<STR_LIT>")<EOL>image = "<STR_LIT>" %image<EOL>tdata.add_item(added.image, image)<EOL>bot.info('<STR_LIT>' %(SREGISTRY_STORAGE,<EOL>endpoint, image))<EOL>transfer_result = self.transfer_client.submit_transfer(tdata)<EOL>bot.info(transfer_result['<STR_LIT:message>'])<EOL>return transfer_result<EOL>

push an image to Globus endpoint. In this case, the name is the
       globus endpoint id and path.

       --name <endpointid>:/path/for/image


<EOL>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._init_transfer_client()<EOL><DEDENT>if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>endpoint, remote = self._parse_endpoint_name(image)<EOL>source = self.transfer_client.get_endpoint(endpoint)<EOL>name = os.path.basename(remote)<EOL>q = parse_image_name(name, default_collection=source['<STR_LIT:name>'])<EOL>endpoints = self._get_endpoints()<EOL>if len(endpoints['<STR_LIT>']) == <NUM_LIT:0>:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>) <EOL><DEDENT>dest = None<EOL>for eid,contender in endpoints['<STR_LIT>'].items():<EOL><INDENT>if contender['<STR_LIT>'] is True:<EOL><INDENT>dest = contender<EOL>break<EOL><DEDENT><DEDENT>if dest is None:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>base = self._get_endpoint_path(dest['<STR_LIT:id>'])<EOL>storage_folder = '<STR_LIT>' %(base, q['<STR_LIT>'])<EOL>self._create_endpoint_folder(dest['<STR_LIT:id>'], storage_folder)<EOL>label = "<STR_LIT>"<EOL>tdata = globus_sdk.TransferData(self.transfer_client, <EOL>source['<STR_LIT:id>'],<EOL>dest['<STR_LIT:id>'],<EOL>label=label,<EOL>sync_level="<STR_LIT>")<EOL>image = os.path.join(base, q['<STR_LIT>'])<EOL>tdata.add_item(remote, image)<EOL>bot.info('<STR_LIT>' %q['<STR_LIT>'])<EOL>transfer_result = self.transfer_client.submit_transfer(tdata)<EOL>bot.info(transfer_result['<STR_LIT:message>'])<EOL>finished.append(transfer_result)<EOL><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from a Globus endpoint. The user must have the default
       local endpoint set up. For example:

       6881ae2e-db26-11e5-9772-22000b9da45e:.singularity/shub/sherlock_vep.simg

    Parameters
    ==========
    images: refers to the globus endpoint id and image path.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add(). For globus this is the only option, and
          we don't have control over when this happens.

    Returns
    =======
    finished: a single container path, or list of paths


<EOL>if query is None:<EOL><INDENT>if args.endpoint is None:<EOL><INDENT>bot.info('<STR_LIT>')<EOL>return self._list_endpoints()<EOL><DEDENT>else:<EOL><INDENT>return self._list_endpoint(args.endpoint)<EOL><DEDENT><DEDENT>if args.endpoint is None:<EOL><INDENT>bot.info('<STR_LIT>')<EOL>return self._list_endpoints(query)<EOL><DEDENT>return self._list_endpoint(endpoint=args.endpoint, <EOL>query=query)<EOL>

query will show images determined by the extension of img
       or simg.

       Parameters
       ==========
       query: the container name (path) or uri to search for
       args.endpoint: can be an endpoint id and optional path, e.g.:

             --endpoint  6881ae2e-db26-11e5-9772-22000b9da45e:.singularity'
             --endpoint  6881ae2e-db26-11e5-9772-22000b9da45e'

       if not defined, we show the user endpoints to choose from

       Usage
       =====
       If endpoint is defined with a query, then we search the given endpoint
       for a container of interested (designated by ending in .img or .simg

       If no endpoint is provided but instead just a query, we use the query
       to search endpoints.


bot.info('<STR_LIT>')<EOL>endpoints = self._get_endpoints(query)<EOL>bot.custom(prefix="<STR_LIT>", message="<STR_LIT>", color="<STR_LIT>")<EOL>rows = []<EOL>for kind,eps in endpoints.items():<EOL><INDENT>for epid,epmeta in eps.items():<EOL><INDENT>rows.append([epid, '<STR_LIT>' %kind, epmeta['<STR_LIT:name>']])<EOL><DEDENT><DEDENT>bot.table(rows)<EOL>return rows<EOL>

list all endpoints, providing a list of endpoints to the user to
       better filter the search. This function takes no arguments,
       as the user has not provided an endpoint id or query.


def list_endpoint(self, endpoint, query=None):

if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._init_transfer_client()<EOL><DEDENT>endpoint, path = self._parse_endpoint_name(endpoint)<EOL>try:<EOL><INDENT>result = self.transfer_client.operation_ls(endpoint, path=path)<EOL><DEDENT>except TransferAPIError as err:<EOL><INDENT>bot.custom(prefix='<STR_LIT>', message=err, color='<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>rows = []<EOL>for filey in result:<EOL><INDENT>name = filey['<STR_LIT:name>']<EOL>if query is None or query in name:<EOL><INDENT>if name.endswith('<STR_LIT>'):<EOL><INDENT>name = bot.addColor('<STR_LIT>',name)<EOL><DEDENT>rows.append([filey['<STR_LIT:type>'],<EOL>filey['<STR_LIT>'],<EOL>str(filey['<STR_LIT:size>']),<EOL>name ])<EOL><DEDENT><DEDENT>if len(rows) > <NUM_LIT:0>:<EOL><INDENT>rows = [["<STR_LIT:type>","<STR_LIT>","<STR_LIT>","<STR_LIT>"]] + rows<EOL>bot.custom(prefix="<STR_LIT>" %path, message='<STR_LIT>', color="<STR_LIT>")<EOL>bot.table(rows)<EOL><DEDENT>else:<EOL><INDENT>bot.info('<STR_LIT>')<EOL><DEDENT>return rows<EOL>

An endpoint is required here to list files within. Optionally, we can
       take a path relative to the endpoint root.

       Parameters
       ==========
       endpoint: a single endpoint ID or an endpoint id and relative path.
                 If no path is provided, we use '', which defaults to scratch.

       query: if defined, limit files to those that have query match


self._client = globus_sdk.NativeAppAuthClient(self._client_id)<EOL>self._load_secrets()<EOL>

init_ cliends will obtain the tranfer and access tokens, and then
           use them to create a transfer client.


<EOL>self.auth = self._get_and_update_setting('<STR_LIT>')<EOL>self.transfer = self._get_and_update_setting('<STR_LIT>')<EOL>

load the secrets credentials file with the Globus OAuthTokenResponse


<EOL>needs_update = True<EOL>if self.auth is not None:<EOL><INDENT>if self.auth['<STR_LIT>'] > time.time():<EOL><INDENT>needs_update = False<EOL><DEDENT><DEDENT>return needs_update<EOL>

return boolean True or False if the client tokens (self._auth and
           self._transfer) need updating.


self._client.oauth2_start_flow(refresh_tokens=True)<EOL>authorize_url = self._client.oauth2_get_authorize_url()<EOL>print('<STR_LIT>'.format(authorize_url))<EOL>auth_code = raw_input(<EOL>'<STR_LIT>').strip()<EOL>self._response = self._client.oauth2_exchange_code_for_tokens(auth_code)<EOL>self.auth = self._response.by_resource_server['<STR_LIT>']<EOL>self.transfer = self._response.by_resource_server['<STR_LIT>']<EOL>self._update_setting('<STR_LIT>', self.transfer)<EOL>self._update_setting('<STR_LIT>', self.auth)<EOL>

Present the client with authentication flow to get tokens from code.
           This simply updates the client _response to be used to get tokens
           for auth and transfer (both use access_token as index). We call
           this not on client initialization, but when the client is actually
           needed.


parts = [x for x in endpoint.split('<STR_LIT::>') if x]<EOL>endpoint = parts[<NUM_LIT:0>]<EOL>if len(parts) == <NUM_LIT:1>:<EOL><INDENT>path = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>path = '<STR_LIT:/>'.join(parts[<NUM_LIT:1>:])<EOL><DEDENT>return endpoint, path<EOL>

split an endpoint name by colon, as the user can provide an
       endpoint name separated from a path:

       Parameters
       ==========
       endpoint 12345:/path/on/remote


def create_endpoint_cache(self, <EOL>endpoint_id, <EOL>cache='<STR_LIT>'):

self._create_endpoint_folder(endpoint_id, cache)<EOL>

create a directory for sregistry in the user's
       base folder to share images.

       Parameters
       ==========
       endpoint_id: the endpoint id parameters
       cache: the relative path for the images cache folder at the
              root of the endpoint


def create_endpoint_folder(self, endpoint_id, folder):

try:<EOL><INDENT>res = self.transfer_client.operation_mkdir(endpoint_id, folder)<EOL>bot.info("<STR_LIT>" %(res['<STR_LIT:message>'], folder))<EOL><DEDENT>except TransferAPIError:<EOL><INDENT>bot.info('<STR_LIT>' %folder)<EOL><DEDENT>

create an endpoint folder, catching the error if it exists.

       Parameters
       ==========
       endpoint_id: the endpoint id parameters
       folder: the relative path of the folder to create


config = os.path.expanduser("<STR_LIT>")<EOL>if not os.path.exists(config):<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>path = None<EOL>config = [x.split('<STR_LIT:U+002C>')[<NUM_LIT:0>] for x in read_file(config)]<EOL>for path in config:<EOL><INDENT>if os.path.exists(path):<EOL><INDENT>break<EOL><DEDENT><DEDENT>if path is None:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return path<EOL>

return the first fullpath to a folder in the endpoint based on
       expanding the user's home from the globus config file. This
       function is fragile but I don't see any other way to do it.

       Parameters
       ==========
       endpoint_id: the endpoint id to look up the path for


if self._tokens_need_update():<EOL><INDENT>self._update_tokens()<EOL><DEDENT>access_token = self.transfer['<STR_LIT>']<EOL>authorizer = globus_sdk.RefreshTokenAuthorizer(<EOL>self.transfer['<STR_LIT>'],<EOL>self._client, <EOL>access_token=self.transfer['<STR_LIT>'], <EOL>expires_at=self.transfer['<STR_LIT>'])<EOL>self.transfer_client = globus_sdk.TransferClient(authorizer=authorizer)<EOL>


endpoint = None<EOL>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._init_transfer_client()<EOL><DEDENT>try:<EOL><INDENT>endpoint = self.transfer_client.get_endpoint(endpoint_id).data<EOL><DEDENT>except TransferAPIError:<EOL><INDENT>bot.info('<STR_LIT>' %endpoint_id)<EOL><DEDENT>return endpoint<EOL>

use a transfer client to get a specific endpoint based on an endpoint id.

       Parameters
       ==========
       endpoint_id: the endpoint_id to retrieve


self.endpoints = {}<EOL>if not hasattr(self, '<STR_LIT>'):<EOL><INDENT>self._init_transfer_client()<EOL><DEDENT>scopes = {'<STR_LIT>':None,<EOL>'<STR_LIT>': None}<EOL>if query is not None:<EOL><INDENT>scopes.update({'<STR_LIT:all>': query})<EOL><DEDENT>for scope, q in scopes.items():  <EOL><INDENT>self.endpoints[scope] = {}      <EOL>for ep in self.transfer_client.endpoint_search(q, filter_scope=scope):<EOL><INDENT>ep = ep.__dict__['<STR_LIT>']<EOL>self.endpoints[scope][ep['<STR_LIT:id>']] = ep<EOL><DEDENT><DEDENT>if len(self.endpoints['<STR_LIT>']) == <NUM_LIT:0>:<EOL><INDENT>bot.warning('<STR_LIT>')<EOL>bot.warning('<STR_LIT>')<EOL><DEDENT>return self.endpoints<EOL>

use a transfer client to get endpoints. If a search term is included,
       we use it to search a scope of "all" in addition to personal and shared
       endpoints. Endpoints are organized
       by type (my-endpoints, shared-with-me, optionally all) and then id.

       Parameters
       ==========
       query: an endpoint search term to add to a scope "all" search. If not
              defined, no searches are done with "all"


path = os.path.abspath(path)<EOL>bot.debug("<STR_LIT>" % path)<EOL>if not os.path.exists(path):<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name(remove_uri(name),tag=tag)<EOL>if names['<STR_LIT:version>'] is None:<EOL><INDENT>version = get_image_hash(path)<EOL>names = parse_image_name(remove_uri(name), tag=tag, version=version)    <EOL><DEDENT>metadata = self.get_metadata(path, names=names)<EOL>if "<STR_LIT:data>" in metadata:<EOL><INDENT>metadata = metadata['<STR_LIT:data>']<EOL><DEDENT>metadata.update(names)<EOL>manifest = self._upload(source=path, <EOL>destination=names['<STR_LIT>'],<EOL>metadata=metadata)<EOL>print(manifest['<STR_LIT>'])<EOL>

push an image to Google Cloud Storage, meaning uploading it

    path: should correspond to an absolte image path (or derive it)
    name: should be the complete uri that the user has requested to push.
    tag: should correspond with an image tag. This is provided to mirror Docker


@retry(wait_exponential_multiplier=<NUM_LIT:1000>, wait_exponential_max=<NUM_LIT>)<EOL>def upload(self, source, destination, chunk_size = <NUM_LIT:2> * <NUM_LIT> * <NUM_LIT>, metadata=None):

env = '<STR_LIT>'<EOL>keep_private = self._get_and_update_setting(env)<EOL>media = MediaFileUpload(source, chunksize=chunk_size, resumable=True)<EOL>request = self._storage_service.objects().insert(bucket=self._bucket_name, <EOL>name=destination,<EOL>media_body=media)<EOL>response = None<EOL>total = request.resumable._size / (<NUM_LIT>*<NUM_LIT>)<EOL>bar = ProgressBar(expected_size=total, filled_char='<STR_LIT:=>', hide=self.quiet)<EOL>while response is None:<EOL><INDENT>error = None<EOL>try:<EOL><INDENT>progress, response = request.next_chunk()<EOL>if progress:<EOL><INDENT>bar.show(progress.resumable_progress / (<NUM_LIT>*<NUM_LIT>))<EOL><DEDENT><DEDENT>except:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>blob = self._bucket.blob(destination)<EOL>if blob.exists():<EOL><INDENT>if not keep_private:<EOL><INDENT>blob.make_public()<EOL><DEDENT>if metadata is not None:<EOL><INDENT>body = prepare_metadata(metadata)<EOL>blob.metadata = metadata   <EOL>blob._properties['<STR_LIT>'] = metadata<EOL>blob.patch()<EOL><DEDENT><DEDENT>return response<EOL>

upload a file from a source to a destination. The client is expected
       to have a bucket (self._bucket) that is created when instantiated.

       This would be the method to do the same using the storage client,
       but not easily done for resumable

       blob = self._bucket.blob(destination)
       blob.upload_from_filename(filename=source,
                                 content_type="application/zip",
                                 client=self._service)

       url = blob.public_url
       if isinstance(url, six.binary_type):
           url = url.decode('utf-8')

       return url


if not isinstance(images,list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' %len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>q = parse_image_name(remove_uri(image))<EOL>bot.info('<STR_LIT>' %(q['<STR_LIT>'],self._bucket_name))<EOL>matches = self._container_query(q['<STR_LIT>'], quiet=True)<EOL>if len(matches) == <NUM_LIT:0>:<EOL><INDENT>bot.info('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>if file_name is None:<EOL><INDENT>file_name = q['<STR_LIT>'].replace('<STR_LIT:/>','<STR_LIT:->')<EOL><DEDENT>image = matches[<NUM_LIT:0>]<EOL>image_file = self.download(url=image.media_link,<EOL>file_name=file_name,<EOL>show_progress=True)<EOL>if save is True:<EOL><INDENT>image_uri = q['<STR_LIT>']<EOL>if "<STR_LIT>" in image.metadata:<EOL><INDENT>image_uri = image.metadata['<STR_LIT>']<EOL><DEDENT>metadata = image.metadata<EOL>metadata['<STR_LIT>'] = image.self_link<EOL>container = self.add(image_path = image_file,<EOL>image_uri = image_uri,<EOL>metadata = metadata,<EOL>url = image.media_link)<EOL>image_file = container.image<EOL><DEDENT>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' %image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL>finished.append(image_file)<EOL><DEDENT><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from google storage, based on the identifier

    Parameters
    ==========
    images: refers to the uri given by the user to pull in the format
    <collection>/<namespace>. You should have an API that is able to
    retrieve a container based on parsing this uri.
    file_name: the user's requested name for the file. It can
               optionally be None if the user wants a default.
    save: if True, you should save the container to the database
          using self.add()

    Returns
    =======
    finished: a single container path, or list of paths


def list_builders(self, project=None, zone='<STR_LIT>'):

builders = []<EOL>instances = self._get_instances(project, zone)<EOL>for instance in instances['<STR_LIT>']:<EOL><INDENT>builders.append([instance['<STR_LIT:name>'], instance['<STR_LIT:status>']])<EOL><DEDENT>bot.info("<STR_LIT>" %(len(builders)))<EOL>bot.table(builders)<EOL>bot.newline()<EOL>

list builders, or instances for the project. They should start with
       sregistry-builder

       Parameters
       ==========
       project: specify a project, will default to environment first
       zone: the zone to use, defaults to us-west1-a if environment not set


configs = self._get_templates()<EOL>rows = []<EOL>if name:<EOL><INDENT>matches = self._load_templates(name)<EOL>bot.debug('<STR_LIT>' %(len(matches), name))<EOL>for match in matches:<EOL><INDENT>print(json.dumps(match, indent=<NUM_LIT:4>, sort_keys=True))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>for config in configs['<STR_LIT:data>']:<EOL><INDENT>rows.append([config['<STR_LIT:name>']])<EOL><DEDENT>bot.table(rows)<EOL><DEDENT>

list templates in the builder bundle library. If a name is provided,
       look it up

       Parameters
       ==========
       name: the name of a template to look up


base = '<STR_LIT>'<EOL>base = self._get_and_update_setting('<STR_LIT>', base)<EOL>base = "<STR_LIT>" %base<EOL>return self._get(base)<EOL>

list templates in the builder bundle library. If a name is provided,
       look it up


configs = self._get_templates()<EOL>templates = []<EOL>matches = [x for x in configs['<STR_LIT:data>'] if name in x['<STR_LIT:name>']]<EOL>if len(matches) > <NUM_LIT:0>:<EOL><INDENT>for match in matches:<EOL><INDENT>response = self._get(match['<STR_LIT:id>'])<EOL>templates.append(response)<EOL><DEDENT>return templates<EOL><DEDENT>bot.info('<STR_LIT>' %name)<EOL>

load a particular template based on a name. We look for a name IN data,
       so the query name can be a partial string of the full name.

       Parameters
       ==========
       name: the name of a template to look up


def get_instances(self, project=None, zone='<STR_LIT>'):

project = self._get_project(project)<EOL>zone = self._get_zone(zone)<EOL>return self._compute_service.instances().list(project=project, <EOL>zone=zone).execute()<EOL>

get instances will return the (unparsed) list of instances, for
       functions for the user. This is primarily used by get_builders
       to print a list of builder instances.

       Parameters
       ==========
       project: specify a project, will default to environment first
       zone: the zone to use, defaults to us-west1-a if environment not set


def get_ipaddress(self, name, retries=<NUM_LIT:3>, delay=<NUM_LIT:3>):

for rr in range(retries):<EOL><INDENT>instances = self._get_instances()<EOL>for instance in instances['<STR_LIT>']:<EOL><INDENT>if instance['<STR_LIT:name>'] == name:<EOL><INDENT>for network in instance['<STR_LIT>']:<EOL><INDENT>if network['<STR_LIT:name>'] == '<STR_LIT>':<EOL><INDENT>for subnet in network['<STR_LIT>']:<EOL><INDENT>if subnet['<STR_LIT:name>'] == '<STR_LIT>':<EOL><INDENT>if '<STR_LIT>' in subnet:<EOL><INDENT>return subnet['<STR_LIT>']<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>sleep(delay) <EOL><DEDENT>bot.warning('<STR_LIT>')<EOL>

get the ip_address of an inserted instance. Will try three times with
       delay to give the instance time to start up.

       Parameters
       ==========
       name: the name of the instance to get the ip address for.
       retries: the number of retries before giving up
       delay: the delay between retry

       Note from @vsoch: this function is pretty nasty.


<EOL>if isinstance(config, dict):<EOL><INDENT>bot.debug('<STR_LIT>')<EOL>return config<EOL><DEDENT>if config is None:<EOL><INDENT>config = self._get_and_update_setting('<STR_LIT>',<EOL>'<STR_LIT>')<EOL><DEDENT>elif os.path.exists(config):<EOL><INDENT>return read_json(config)<EOL><DEDENT>configs = self._load_templates(config)<EOL>if configs is None:<EOL><INDENT>bot.error('<STR_LIT>' %name)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>bot.info('<STR_LIT>' %config)<EOL>config = configs[<NUM_LIT:0>]<EOL>return config<EOL>

load a google compute config, meaning that we have the following cases:

       1. the user has not provided a config file directly, we look in env.
       2. the environment is not set, so we use a reasonable default
       3. if the final string is not found as a file, we look for it in library
       4. we load the library name, or the user file, else error

       Parameters
       ==========
       config: the config file the user has provided, or the library URI


def setup_build(self, name, repo, config, tag=None, commit=None,<EOL>recipe="<STR_LIT>", startup_script=None):

manager = self._get_and_update_setting('<STR_LIT>', '<STR_LIT>')<EOL>startup_script = get_build_template(startup_script, manager)<EOL>config = self._load_build_config(config)<EOL>if not config:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>self._client_tagged(config['<STR_LIT:data>']['<STR_LIT>'])<EOL>defaults = config['<STR_LIT:data>']['<STR_LIT>']<EOL>selfLink = config['<STR_LIT>']['<STR_LIT>']<EOL>builder_repo = config['<STR_LIT:data>']['<STR_LIT>']<EOL>builder_bundle = config['<STR_LIT:data>']['<STR_LIT:path>']<EOL>builder_id = config['<STR_LIT:data>']['<STR_LIT:id>']<EOL>config = config['<STR_LIT:data>']['<STR_LIT>']<EOL>image_project = defaults.get('<STR_LIT>', '<STR_LIT>')<EOL>image_family = defaults.get('<STR_LIT>', '<STR_LIT>')<EOL>instance_name = "<STR_LIT>" %(name.replace('<STR_LIT:/>','<STR_LIT:->'), selfLink)<EOL>robot_name = RobotNamer().generate()<EOL>project = self._get_project()<EOL>zone = self._get_zone()<EOL>machine_type = defaults.get('<STR_LIT>', '<STR_LIT>')<EOL>machine_type = "<STR_LIT>" %(zone, machine_type)<EOL>disk_size = defaults.get('<STR_LIT>', '<STR_LIT:100>')<EOL>image_response = self._compute_service.images().getFromFamily(<EOL>project=image_project, <EOL>family=image_family).execute()<EOL>source_disk_image = image_response['<STR_LIT>']<EOL>storage_bucket = self._bucket_name<EOL>config['<STR_LIT:name>'] = robot_name<EOL>config['<STR_LIT:description>'] = instance_name<EOL>config['<STR_LIT>'] = machine_type<EOL>config['<STR_LIT>'].append({<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT>": True,<EOL>"<STR_LIT>": { '<STR_LIT>': source_disk_image,<EOL>'<STR_LIT>': disk_size }<EOL>})<EOL>metadata = {'<STR_LIT>': <EOL>[{ '<STR_LIT:key>': '<STR_LIT>',<EOL>'<STR_LIT:value>': startup_script },<EOL>{ '<STR_LIT:key>':'<STR_LIT>',<EOL>'<STR_LIT:value>':self._bucket_name }]}<EOL>defaults = setconfig(defaults, '<STR_LIT>', repo)<EOL>defaults = setconfig(defaults, '<STR_LIT>', name)<EOL>defaults = setconfig(defaults, '<STR_LIT>', commit)<EOL>defaults = setconfig(defaults, '<STR_LIT>', "<STR_LIT>")<EOL>defaults = setconfig(defaults, '<STR_LIT>', tag)<EOL>defaults = setconfig(defaults, '<STR_LIT>', builder_repo)<EOL>defaults = setconfig(defaults, '<STR_LIT>')<EOL>defaults = setconfig(defaults, '<STR_LIT>', "<STR_LIT>")<EOL>defaults = setconfig(defaults, '<STR_LIT>', "<STR_LIT>")<EOL>defaults = setconfig(defaults, '<STR_LIT>', builder_id)<EOL>defaults = setconfig(defaults, '<STR_LIT>', builder_bundle)<EOL>defaults = setconfig(defaults, '<STR_LIT>', "<STR_LIT:4>")<EOL>defaults = setconfig(defaults, '<STR_LIT>', "<STR_LIT>")<EOL>defaults = setconfig(defaults, '<STR_LIT>', recipe)<EOL>defaults = setconfig(defaults, '<STR_LIT>')<EOL>defaults = setconfig(defaults, '<STR_LIT>')<EOL>defaults = setconfig(defaults,'<STR_LIT>', <EOL>'<STR_LIT>')<EOL>seen = ['<STR_LIT>', '<STR_LIT>']<EOL>for key, value in defaults.items():<EOL><INDENT>if value not in seen:<EOL><INDENT>entry = { "<STR_LIT:key>": key, '<STR_LIT:value>': value }<EOL>metadata['<STR_LIT>'].append(entry)<EOL>seen.append(key)<EOL><DEDENT><DEDENT>config['<STR_LIT>'] = metadata<EOL>return config<EOL>

setup the build based on the selected configuration file, meaning
       producing the configuration file filled in based on the user's input

       Parameters
       ==========
       config: the complete configuration file provided by the client
       template: an optional custom start script to use
       tag: a user specified tag for the build, derived from uri or manual
       recipe: a recipe, if defined, overrides recipe set in config.
       commit: a commit to check out, if needed
       start_script: the start script to use, if not defined
                     defaults to apt (or manager) base in main/templates/build


lookup[key] = value or lookup.get(key)<EOL>return lookup<EOL>

setconfig will update a lookup to give priority based on the following:

       1. If both values are None, we set the value to None
       2. If the currently set (the config.json) is set but not runtime, use config
       3. If the runtime is set but not config.json, we use runtime
       4. If both are set, we use runtime


project = self._get_project()<EOL>zone = self._get_zone()<EOL>bot.custom(prefix='<STR_LIT>', message=config['<STR_LIT:name>'], color="<STR_LIT>")<EOL>bot.info(config['<STR_LIT:description>'])<EOL>response = self._compute_service.instances().insert(project=project,<EOL>zone=zone,<EOL>body=config).execute()<EOL>ipaddress = self._get_ipaddress(config['<STR_LIT:name>'])<EOL>bot.info('<STR_LIT>' %ipaddress)<EOL>bot.info('<STR_LIT>')<EOL>return response<EOL>

run a build, meaning inserting an instance. Retry if there is failure

       Parameters
       ==========
       config: the configuration dictionary generated by setup_build


if query is not None:<EOL><INDENT>return self._container_query(query)<EOL><DEDENT>return self._search_all()<EOL>

query a bucket for images that match a particular pattern. If no
       query is provided, all images in the bucket are listed that have type
       "container" in the metadata and client "sregistry"


results = []<EOL>for image in self._bucket.list_blobs():<EOL><INDENT>if image.metadata is not None:<EOL><INDENT>if "<STR_LIT:type>" in image.metadata:<EOL><INDENT>if image.metadata['<STR_LIT:type>'] == "<STR_LIT>":<EOL><INDENT>results.append(image)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL><DEDENT>return results<EOL>

return a list of containers, determined by finding the metadata field
       "type" with value "container." We alert the user to no containers
       if results is empty, and exit

       {'metadata': {'items':
                              [
                               {'key': 'type', 'value': 'container'}, ...
                              ]
                    }
       }


results = self._list_containers()<EOL>bot.info("<STR_LIT>" %self._bucket_name)<EOL>rows = []<EOL>for i in results:<EOL><INDENT>size = round(i.size / (<NUM_LIT>*<NUM_LIT>))<EOL>size = ("<STR_LIT>" %size).rjust(<NUM_LIT:10>)<EOL>rows.append([size,i.metadata['<STR_LIT>']])<EOL><DEDENT>bot.table(rows)<EOL>return rows<EOL>

a "list all" search that doesn't require a query. Here we return to
       the user all objects that have custom metadata value of "container"

       IMPORTANT: the upload function adds this metadata. For a container to
       be found by the client, it must have the type as container in metadata.


results = self._list_containers()<EOL>matches = []<EOL>for result in results:<EOL><INDENT>for key,val in result.metadata.items():<EOL><INDENT>if query in val and result not in matches:<EOL><INDENT>matches.append(result)<EOL><DEDENT><DEDENT><DEDENT>if not quiet:<EOL><INDENT>bot.info("<STR_LIT>" %(self._bucket_name,len(matches)))<EOL>for image in matches:<EOL><INDENT>size = round(image.size / (<NUM_LIT>*<NUM_LIT>))<EOL>bot.custom(prefix=image.name, color="<STR_LIT>")<EOL>bot.custom(prefix='<STR_LIT>', message=image.id)<EOL>bot.custom(prefix='<STR_LIT>', message=image.metadata['<STR_LIT>'])<EOL>bot.custom(prefix='<STR_LIT>', message=image.updated)<EOL>bot.custom(prefix='<STR_LIT>',  message='<STR_LIT>' %(size))<EOL>bot.custom(prefix='<STR_LIT>', message=image.md5_hash)<EOL>bot.newline()<EOL><DEDENT><DEDENT>return matches<EOL>

search for a specific container.
    This function would likely be similar to the above, but have different
    filter criteria from the user (based on the query)


bot.info('<STR_LIT>' %self._bucket_name)<EOL>

add the bucket name to be printed to the user at appropriate times


env = '<STR_LIT>'<EOL>self._secrets = self._get_and_update_setting(env)<EOL>if self._secrets is None:<EOL><INDENT>bot.error('<STR_LIT>' %env)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>

The user is required to have an application secrets file in his
           or her environment. The information isn't saved to the secrets
           file, but the client exists with error if the variable isn't found.


<EOL>self._get_services()<EOL>env = '<STR_LIT>'<EOL>self._bucket_name = self._get_and_update_setting(env)<EOL>if self._bucket_name is None:<EOL><INDENT>self._bucket_name = '<STR_LIT>' %os.environ['<STR_LIT>']<EOL><DEDENT>self._get_bucket()<EOL>

init client will check if the user has defined a bucket that
           differs from the default, use the application credentials to
           get the bucket, and then instantiate the client.


self._bucket_service = storage.Client()<EOL>creds = GoogleCredentials.get_application_default()<EOL>self._storage_service = discovery_build('<STR_LIT>', version, credentials=creds)<EOL>self._compute_service = discovery_build('<STR_LIT>', version, credentials=creds)<EOL>

get version 1 of the google compute and storage service

        Parameters
        ==========
        version: version to use (default is v1)


<EOL>try:<EOL><INDENT>self._bucket = self._bucket_service.get_bucket(self._bucket_name)<EOL><DEDENT>except google.cloud.exceptions.NotFound:<EOL><INDENT>self._bucket = self._bucket_service.create_bucket(self._bucket_name)<EOL><DEDENT>except:<EOL><INDENT>bot.error('<STR_LIT>' %self._bucket_name)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>return self._bucket<EOL>

get a bucket based on a bucket name. If it doesn't exist, create it.


return self._required_get_and_update('<STR_LIT>', project)<EOL>

get project returns the active project, and exists if not found.

           Parameters
           ==========
           project: a project to default to, if not found in the environment
           zone: a default zone, will be us-west1-a by default


return self._get_and_update_setting('<STR_LIT>', zone)<EOL>

get zone returns the zone set in the environment, or the default

           Parameters
           ==========
           zone: a default zone, will be us-west1-a by default


pairs = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': [{<EOL>'<STR_LIT:key>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT>'<EOL>}<EOL>]<EOL>}<EOL>}<EOL>for key,val in metadata.items():<EOL><INDENT>if not isinstance(val,dict) and not isinstance(val,list):<EOL><INDENT>pairs['<STR_LIT>']['<STR_LIT>'].append({'<STR_LIT:key>':key,'<STR_LIT:value>':val})<EOL><DEDENT>elif isinstance(val,dict):            <EOL><INDENT>for k,v in val.items():<EOL><INDENT>if not isinstance(v,dict) and not isinstance(v,list):<EOL><INDENT>pairs['<STR_LIT>']['<STR_LIT>'].append({'<STR_LIT:key>':k,'<STR_LIT:value>':v})<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return pairs<EOL>

prepare a key/value list of metadata for the request. The metadata
       object that comes in is only parsed one level.


base = get_installdir()<EOL>if name is None:<EOL><INDENT>name = "<STR_LIT>" %(base,<EOL>manager)<EOL><DEDENT>if os.path.exists(name):<EOL><INDENT>bot.debug("<STR_LIT>" %name)<EOL>return '<STR_LIT>'.join(read_file(name)) <EOL><DEDENT>bot.warning("<STR_LIT>" %name)<EOL>

get a particular build template, by default we return templates
       that are based on package managers.

       Parameters
       ==========
       name: the full path of the template file to use.
       manager: the package manager to use in the template (yum or apt)


path = os.path.abspath(path)<EOL>bot.debug("<STR_LIT>" % path)<EOL>if not os.path.exists(path):<EOL><INDENT>bot.error('<STR_LIT>' %path)<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>names = parse_image_name(remove_uri(name), tag=tag)<EOL>file_size = os.path.getsize(path)<EOL>chunk_size = <NUM_LIT:4> * <NUM_LIT> * <NUM_LIT><EOL>storage_path = "<STR_LIT>" %names['<STR_LIT>']<EOL>collection = self._get_or_create_collection(names['<STR_LIT>'])<EOL>image_name = os.path.basename(names['<STR_LIT>'])<EOL>progress = <NUM_LIT:0><EOL>bot.show_progress(progress, file_size, length=<NUM_LIT>)<EOL>with open(path, '<STR_LIT:rb>') as F:<EOL><INDENT>self.conn.put_object(names['<STR_LIT>'], image_name,<EOL>contents= F.read(),<EOL>content_type='<STR_LIT>')<EOL><DEDENT>bot.show_progress(iteration=file_size,<EOL>total=file_size,<EOL>length=<NUM_LIT>,<EOL>carriage_return=True)<EOL>sys.stdout.write('<STR_LIT:\n>')<EOL>

push an image to your Storage. If the collection doesn't exist,
       it is created.

       Parameters
       ==========
       path: should correspond to an absolute image path (or derive it)
       name: should be the complete uri that the user has requested to push.
       tag: should correspond with an image tag. This is provided to mirror Docker


force = False<EOL>if "<STR_LIT>" in kwargs:<EOL><INDENT>force = kwargs['<STR_LIT>']<EOL><DEDENT>if not isinstance(images, list):<EOL><INDENT>images = [images]<EOL><DEDENT>bot.debug('<STR_LIT>' % len(images))<EOL>finished = []<EOL>for image in images:<EOL><INDENT>names = parse_image_name(remove_uri(image))<EOL>collection = self._get_collection(names['<STR_LIT>'])<EOL>if collection is None:<EOL><INDENT>bot.error('<STR_LIT>' % names['<STR_LIT>'])<EOL>collections = self.get_collections()<EOL>if collections:<EOL><INDENT>bot.info('<STR_LIT>' %'<STR_LIT:\n>'.join(collections))<EOL><DEDENT>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>image_name = os.path.basename(names['<STR_LIT>'])<EOL>try:<EOL><INDENT>obj_tuple = self.conn.get_object(names['<STR_LIT>'], image_name)<EOL><DEDENT>except ClientException:<EOL><INDENT>bot.exit('<STR_LIT>' % names['<STR_LIT>'])<EOL><DEDENT>if names['<STR_LIT:version>'] == None:<EOL><INDENT>names['<STR_LIT:version>'] = obj_tuple[<NUM_LIT:0>]['<STR_LIT>']<EOL><DEDENT>if file_name is None:<EOL><INDENT>file_name = self._get_storage_name(names)<EOL><DEDENT>if os.path.exists(file_name) and force is False:<EOL><INDENT>bot.error('<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>) <EOL><DEDENT>with open(file_name, '<STR_LIT:wb>') as filey:<EOL><INDENT>filey.write(obj_tuple[<NUM_LIT:1>])<EOL><DEDENT>if save is True:<EOL><INDENT>names.update(obj_tuple[<NUM_LIT:0>])<EOL>container = self.add(image_path = file_name,<EOL>image_uri = names['<STR_LIT>'],<EOL>metadata = names)<EOL>image_file = container.image<EOL>if os.path.exists(image_file):<EOL><INDENT>bot.debug('<STR_LIT>' %image_file)<EOL>bot.custom(prefix="<STR_LIT>", message=image_file)<EOL>finished.append(image_file)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>bot.error('<STR_LIT>' % path)<EOL><DEDENT><DEDENT>if len(finished) == <NUM_LIT:1>:<EOL><INDENT>finished = finished[<NUM_LIT:0>]<EOL><DEDENT>return finished<EOL>

pull an image from storage using Swift. The image is found based on the
       storage uri

       Parameters
       ==========
       images: refers to the uri given by the user to pull in the format
       <collection>/<namespace>. You should have an API that is able to
       retrieve a container based on parsing this uri.
       file_name: the user's requested name for the file. It can
                  optionally be None if the user wants a default.
       save: if True, you should save the container to the database
             using self.add()

       Returns
       =======
       finished: a single container path, or list of paths


query a Singularity registry for a list of images.
     If query is None, collections are listed.


results = set()<EOL>for container in self.conn.get_account()[<NUM_LIT:1>]:<EOL><INDENT>for result in self.conn.get_container(container['<STR_LIT:name>'])[<NUM_LIT:1>]:<EOL><INDENT>results.add('<STR_LIT>' %(container['<STR_LIT:name>'], result['<STR_LIT:name>']))<EOL><DEDENT><DEDENT>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>bot.info("<STR_LIT>")<EOL>bot.table([[x] for x in list(results)])<EOL>return list(results)<EOL>

a "show all" search that doesn't require a query


results = set()<EOL>query = remove_uri(query)<EOL>for container in self.conn.get_account()[<NUM_LIT:1>]:<EOL><INDENT>for result in self.conn.get_container(container['<STR_LIT:name>'])[<NUM_LIT:1>]:<EOL><INDENT>if query in collection['<STR_LIT:name>']:<EOL><INDENT>results.add('<STR_LIT>' %(container['<STR_LIT:name>'], result['<STR_LIT:name>']))<EOL><DEDENT><DEDENT><DEDENT>if len(results) == <NUM_LIT:0>:<EOL><INDENT>bot.info("<STR_LIT>")<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>bot.info("<STR_LIT>")<EOL>bot.table([list(results)])<EOL>return list(results)<EOL>


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>bot.info('<STR_LIT>' % self.name)<EOL><DEDENT>

if you want to add an extra print (of a parameter, for example)
           for the user when the client initalizes, write it here, eg:
           bot.info('[setting] value')


collections = []<EOL>for container in self.conn.get_account()[<NUM_LIT:1>]:<EOL><INDENT>collections.append(container['<STR_LIT:name>'])<EOL><DEDENT>return collections<EOL>

get a listing of collections that the user has access to.


return self.conn.get_container(name)<EOL>

get a collection name, which corresponds to an upper level
           "container" in ceph storage. If we find and get it, return it.
           If it doesn't exist, return None. To get and create, use
           get_and_create_collection()


try:     <EOL><INDENT>collection = self._get_collection(name)<EOL><DEDENT>except:<EOL><INDENT>bot.info('<STR_LIT>' % name)<EOL>collection = self.conn.put_container(name)<EOL><DEDENT>return collection<EOL>

get or create a collection, meaning that if the get returns
           None, create and return the response to the user.

           Parameters
           ==========
           name: the name of the collection to get (and create)


<EOL>self.config['<STR_LIT>'] = self._required_get_and_update(<EOL>'<STR_LIT>')<EOL>if self.config['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>for envar in ['<STR_LIT>',<EOL>'<STR_LIT>' ]:<EOL><INDENT>self.config[envar] = self._required_get_and_update(envar)<EOL><DEDENT>self.conn = swiftclient.Connection(<EOL>preauthurl=self.config['<STR_LIT>'],<EOL>preauthtoken=self.config['<STR_LIT>']<EOL>)<EOL><DEDENT>elif self.config['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>for envar in ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']:<EOL><INDENT>self.config[envar] = self._required_get_and_update(envar)<EOL><DEDENT>auth_url = '<STR_LIT>' % self.config['<STR_LIT>']<EOL>_os_options = {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>self.conn = swiftclient.Connection(<EOL>user=self.config['<STR_LIT>'],<EOL>key=self.config['<STR_LIT>'],<EOL>os_options=_os_options,<EOL>authurl=auth_url,<EOL>auth_version='<STR_LIT:3>'<EOL>)<EOL><DEDENT>elif self.config['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>for envar in ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']:<EOL><INDENT>self.config[envar] = self._required_get_and_update(envar)<EOL><DEDENT>auth_url = '<STR_LIT>' % self.config['<STR_LIT>']<EOL>_os_options = {<EOL>'<STR_LIT>': self.config['<STR_LIT>'],<EOL>'<STR_LIT>': self.config['<STR_LIT>']<EOL>}<EOL>self.conn = swiftclient.Connection(<EOL>user=self.config['<STR_LIT>'],<EOL>key=self.config['<STR_LIT>'],<EOL>os_options=_os_options,<EOL>authurl=auth_url,<EOL>auth_version='<STR_LIT:2>'<EOL>)<EOL><DEDENT>else:<EOL><INDENT>for envar in ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']:<EOL><INDENT>self.config[envar] = self._required_get_and_update(envar)<EOL><DEDENT>auth_url = '<STR_LIT>' % self.config['<STR_LIT>']<EOL>self.conn = swiftclient.Connection(<EOL>user=self.config['<STR_LIT>'],<EOL>key=self.config['<STR_LIT>'],<EOL>authurl=auth_url,<EOL>)<EOL><DEDENT>

update secrets will look for a user and token in the environment
           If we find the values, cache and continue. Otherwise, exit with error


from sregistry.defaults import ( CREDENTIAL_CACHE, SREGISTRY_CLIENT )<EOL>client_credential_cache = None<EOL>if CREDENTIAL_CACHE is not None: <EOL><INDENT>env = '<STR_LIT>' %SREGISTRY_CLIENT.upper()<EOL>if os.environ.get(env) is not None:<EOL><INDENT>bot.debug('<STR_LIT>' %SREGISTRY_CLIENT)<EOL>CREDENTIAL_CACHE = None<EOL><DEDENT><DEDENT>if CREDENTIAL_CACHE is not None:<EOL><INDENT>if not os.path.exists(CREDENTIAL_CACHE):<EOL><INDENT>mkdir_p(CREDENTIAL_CACHE)<EOL><DEDENT>client_credential_cache = '<STR_LIT>' %(CREDENTIAL_CACHE, SREGISTRY_CLIENT)<EOL><DEDENT>if client_credential_cache is not None:<EOL><INDENT>bot.debug('<STR_LIT>')<EOL><DEDENT>return client_credential_cache<EOL>

if the user has specified settings to provide a cache for credentials
       files, initialize it. The root for the folder is created if it doesn't
       exist. The path for the specific client is returned, and it's
       not assumed to be either a folder or a file (this is up to the
       developer of the client).


client_secrets = { '<STR_LIT>': { '<STR_LIT>': "<STR_LIT>" } }<EOL>return client_secrets<EOL>

return default client secrets, including singularity hub base


def update_client_secrets(backend, updates, secrets=None, save=True):

if secrets is None:<EOL><INDENT>secrets = read_client_secrets()<EOL><DEDENT>if backend not in secrets:<EOL><INDENT>secrets[backend] = {}<EOL><DEDENT>secrets[backend].update(updates)<EOL>if save is True:<EOL><INDENT>secrets_file = get_secrets_file()<EOL>if secrets_file is not None:<EOL><INDENT>write_json(secrets,secrets_file)<EOL><DEDENT><DEDENT>return secrets<EOL>

update client secrets will update the data structure for a particular
       authentication. This should only be used for a (quasi permanent) token
       or similar. The secrets file, if found, is updated and saved by default.


client_secrets = _default_client_secrets()<EOL>secrets = get_secrets_file()<EOL>if secrets is not None:<EOL><INDENT>client_secrets = read_json(secrets)<EOL><DEDENT>else:<EOL><INDENT>from sregistry.defaults import SREGISTRY_CLIENT_SECRETS<EOL>write_json(client_secrets, SREGISTRY_CLIENT_SECRETS)<EOL><DEDENT>return client_secrets<EOL>

for private or protected registries, a client secrets file is required
       to be located at .sregistry. If no secrets are found, we use default
       of Singularity Hub, and return a dummy secrets.


from sregistry.defaults import SREGISTRY_CLIENT_SECRETS<EOL>if os.path.exists(SREGISTRY_CLIENT_SECRETS):<EOL><INDENT>return SREGISTRY_CLIENT_SECRETS<EOL><DEDENT>

Sniff the environment and standard location for client
       secrets file. Otherwise, return None


s = "<STR_LIT>" % (username, password)<EOL>if sys.version_info[<NUM_LIT:0>] >= <NUM_LIT:3>:<EOL><INDENT>s = bytes(s, '<STR_LIT:utf-8>')<EOL>credentials = base64.b64encode(s).decode('<STR_LIT:utf-8>')<EOL><DEDENT>else:<EOL><INDENT>credentials = base64.b64encode(s)<EOL><DEDENT>auth = {"<STR_LIT>": "<STR_LIT>" % credentials}<EOL>return auth<EOL>

generate a base64 encoded header to ask for a token. This means
                base64 encoding a username and password and adding to the
                Authorization header to identify the client.

    Parameters
    ==========
    username: the username
    password: the password


lookup = dict()<EOL>version_file = os.path.join('<STR_LIT>', '<STR_LIT>')<EOL>with open(version_file) as filey:<EOL><INDENT>exec(filey.read(), lookup)<EOL><DEDENT>return lookup<EOL>

get version by way of sregistry.version, returns a
    lookup dictionary with several global variables without
    needing to import singularity


if lookup == None:<EOL><INDENT>lookup = get_lookup()<EOL><DEDENT>install_requires = []<EOL>for module in lookup[key]:<EOL><INDENT>module_name = module[<NUM_LIT:0>]<EOL>module_meta = module[<NUM_LIT:1>]<EOL>if "<STR_LIT>" in module_meta:<EOL><INDENT>dependency = "<STR_LIT>" %(module_name,module_meta['<STR_LIT>'])<EOL><DEDENT>elif "<STR_LIT>" in module_meta:<EOL><INDENT>if module_meta['<STR_LIT>'] == None:<EOL><INDENT>dependency = module_name<EOL><DEDENT>else:<EOL><INDENT>dependency = "<STR_LIT>" %(module_name,module_meta['<STR_LIT>'])<EOL><DEDENT><DEDENT>install_requires.append(dependency)<EOL><DEDENT>return install_requires<EOL>

get requirements, mean reading in requirements and versions from
    the lookup obtained with get_lookup


def handle_cdata(pos, window, charpat, stopchars):

cdata = '<STR_LIT>'<EOL>cursor = start = pos<EOL>try:<EOL><INDENT>while True:<EOL><INDENT>while charpat.match(window[cursor]):<EOL><INDENT>cursor += <NUM_LIT:1><EOL><DEDENT>addchars = window[start:cursor]<EOL>cdata += addchars<EOL>if window[cursor] in stopchars:<EOL><INDENT>return cdata, cursor<EOL><DEDENT>elif window[cursor] == '<STR_LIT:&>':<EOL><INDENT>start = cursor = cursor + <NUM_LIT:1><EOL>if window[cursor] == '<STR_LIT:#>' and window[cursor + <NUM_LIT:1>] == '<STR_LIT:x>':<EOL><INDENT>start = cursor = cursor + <NUM_LIT:2><EOL>while True:<EOL><INDENT>if HEXCHARENTOK.match(window[cursor]):<EOL><INDENT>cursor += <NUM_LIT:1><EOL><DEDENT>elif window[cursor] == '<STR_LIT:;>':<EOL><INDENT>c = chr(int(window[start:cursor], <NUM_LIT:16>))<EOL>if not CHARACTER.match(c):<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format('<STR_LIT:&>' + window[start:cursor] + '<STR_LIT:;>'))<EOL><DEDENT>cdata += c<EOL>break<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(window[cursor]))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>while True:<EOL><INDENT>if NAMEDCHARENTOK.match(window[cursor]):<EOL><INDENT>cursor += <NUM_LIT:1><EOL><DEDENT>elif window[cursor] == '<STR_LIT:;>':<EOL><INDENT>for cn, c in CHARNAMES:<EOL><INDENT>if window[start:cursor] == cn:<EOL><INDENT>cdata += c<EOL>break<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(repr(window[start:cursor])))<EOL><DEDENT>break<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(window[cursor]), error_context(window, start, cursor))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>cursor += <NUM_LIT:1><EOL>start = cursor<EOL><DEDENT><DEDENT>except IndexError:<EOL><INDENT>return None, cursor<EOL><DEDENT>

Return (result, new_position) tuple.
Result is cdata string if possible and None if more input is needed
Or of course bad syntax can raise a RuntimeError


if not isinstance(node, element):<EOL><INDENT>return node.xml_value if outermost else [node.xml_value]<EOL><DEDENT>accumulator = []<EOL>for child in node.xml_children:<EOL><INDENT>if isinstance(child, text):<EOL><INDENT>accumulator.append(child.xml_value)<EOL><DEDENT>elif isinstance(child, element):<EOL><INDENT>accumulator.extend(strval(child, outermost=False))<EOL><DEDENT><DEDENT>if outermost: accumulator = '<STR_LIT>'.join(accumulator)<EOL>return accumulator<EOL>


Append a node as the last child

child - the child to append. If a string, convert to a text node, for convenience


def xml_insert(self, child, index=-<NUM_LIT:1>):

if isinstance(child, str):<EOL><INDENT>child = text(child, parent=self)<EOL><DEDENT>else:<EOL><INDENT>child._xml_parent = weakref.ref(self)<EOL><DEDENT>if index == -<NUM_LIT:1>:<EOL><INDENT>self.xml_children.append(child)<EOL><DEDENT>else:<EOL><INDENT>self.xml_children.insert(index, child)<EOL><DEDENT>return<EOL>


h = expat_callbacks(handler)<EOL>p = xml.parsers.expat.ParserCreate(namespace_separator='<STR_LIT:U+0020>')<EOL>p.StartElementHandler = h.start_element<EOL>p.EndElementHandler = h.end_element<EOL>p.CharacterDataHandler = h.char_data<EOL>p.StartNamespaceDeclHandler = h.start_namespace<EOL>p.EndNamespaceDeclHandler = h.end_namespace<EOL>p.Parse(source)<EOL>return p<EOL>

Convert XML 1.0 to MicroXML

source - XML 1.0 input
handler - MicroXML events handler

Returns uxml, extras

uxml - MicroXML element extracted from the source
extras - information to be preserved but not part of MicroXML, e.g. namespaces


if isinstance(obj, LiteralWrapper):<EOL><INDENT>val = obj.obj<EOL><DEDENT>elif isinstance(obj, Iterable) and not isinstance(obj, str):<EOL><INDENT>val = next(obj, None)<EOL><DEDENT>else:<EOL><INDENT>val = obj<EOL><DEDENT>if val is None:<EOL><INDENT>yield '<STR_LIT>'<EOL><DEDENT>elif isinstance(val, str):<EOL><INDENT>yield val<EOL><DEDENT>elif isinstance(val, node):<EOL><INDENT>yield strval(val)<EOL><DEDENT>elif isinstance(val, int) or isinstance(val, float):<EOL><INDENT>yield str(val)<EOL><DEDENT>elif isinstance(item, bool):<EOL><INDENT>yield '<STR_LIT:true>' if item else '<STR_LIT:false>'<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(val))<EOL><DEDENT>

Cast an arbitrary object or sequence to a string type


if isinstance(obj, LiteralWrapper):<EOL><INDENT>val = obj.obj<EOL><DEDENT>elif isinstance(obj, Iterable) and not isinstance(obj, str):<EOL><INDENT>val = next(obj, None)<EOL><DEDENT>else:<EOL><INDENT>val = obj<EOL><DEDENT>if val is None:<EOL><INDENT>yield <NUM_LIT:0><EOL><DEDENT>elif isinstance(val, str):<EOL><INDENT>yield float(val)<EOL><DEDENT>elif isinstance(val, node):<EOL><INDENT>yield float(strval(val))<EOL><DEDENT>elif isinstance(val, int) or isinstance(val, float):<EOL><INDENT>yield val<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(val))<EOL><DEDENT>

Cast an arbitrary object or sequence to a number type


<EOL>if isinstance(obj, LiteralWrapper):<EOL><INDENT>val = obj.obj<EOL><DEDENT>elif isinstance(obj, Iterable) and not isinstance(obj, str):<EOL><INDENT>val = next(obj, None)<EOL><DEDENT>else:<EOL><INDENT>val = obj<EOL><DEDENT>if val is None:<EOL><INDENT>yield False<EOL><DEDENT>elif isinstance(val, bool):<EOL><INDENT>yield val<EOL><DEDENT>elif isinstance(val, str):<EOL><INDENT>yield bool(str)<EOL><DEDENT>elif isinstance(val, node):<EOL><INDENT>yield True<EOL><DEDENT>elif isinstance(val, float) or isinstance(val, int):<EOL><INDENT>yield bool(val)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(val))<EOL><DEDENT>

Cast an arbitrary sequence to a boolean type


return '<STR_LIT>'.join(_serialize(xp_ast))<EOL>

Serialize an XPath AST as a valid XPath expression.


if hasattr(xp_ast, '<STR_LIT>'):<EOL><INDENT>for tok in xp_ast._serialize():<EOL><INDENT>yield(tok)<EOL><DEDENT><DEDENT>elif isinstance(xp_ast, str):<EOL><INDENT>yield(repr(xp_ast))<EOL><DEDENT>

Generate token strings which, when joined together, form a valid
    XPath serialization of the AST.


yield from self.compute(ctx)<EOL>


p[<NUM_LIT:0>] = ast.BinaryExpression(p[<NUM_LIT:1>], p[<NUM_LIT:2>], p[<NUM_LIT:3>])<EOL>

Expr : Expr OR_OP Expr
     | Expr AND_OP Expr
     | Expr EQUAL_OP Expr
     | Expr REL_OP Expr
     | Expr PLUS_OP Expr
     | Expr MINUS_OP Expr
     | Expr MULT_OP Expr
     | Expr DIV_OP Expr
     | Expr MOD_OP Expr


p[<NUM_LIT:0>] = ast.UnaryExpression(p[<NUM_LIT:1>], p[<NUM_LIT:2>])<EOL>


p[<NUM_LIT:0>] = ast.Sequence([])<EOL>


p[<NUM_LIT:0>] = ast.Sequence(p[<NUM_LIT:2>])<EOL>


p[<NUM_LIT:0>] = [p[<NUM_LIT:1>]]<EOL>


p[<NUM_LIT:0>] = p[<NUM_LIT:1>]<EOL>p[<NUM_LIT:0>].append(p[<NUM_LIT:3>])<EOL>


Expr : Expr PATH_SEP RelativeLocationPath
     | Expr ABBREV_PATH_SEP RelativeLocationPath


p[<NUM_LIT:0>] = p[<NUM_LIT:1>]<EOL>

Expr : RelativeLocationPath
     | AbsoluteLocationPath
     | AbbreviatedAbsoluteLocationPath
     | PredicatedExpression
     | FunctionCall
     | VariableReference


p[<NUM_LIT:0>] = ast.LiteralWrapper(p[<NUM_LIT:1>])<EOL>


p[<NUM_LIT:0>] = ast.AbsolutePath(p[<NUM_LIT:1>])<EOL>


p[<NUM_LIT:0>] = ast.AbsolutePath(p[<NUM_LIT:1>], p[<NUM_LIT:2>])<EOL>

AbsoluteLocationPath : PATH_SEP RelativeLocationPath


AbbreviatedAbsoluteLocationPath : ABBREV_PATH_SEP RelativeLocationPath


p[<NUM_LIT:0>] = p[<NUM_LIT:1>]<EOL>


RelativeLocationPath : RelativeLocationPath PATH_SEP Step
                     | RelativeLocationPath ABBREV_PATH_SEP Step


p[<NUM_LIT:0>] = ast.AbbreviatedStep(p[<NUM_LIT:1>])<EOL>

Step : ABBREV_STEP_SELF
     | ABBREV_STEP_PARENT


<EOL>if p[<NUM_LIT:1>] not in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>raise RuntimeError("<STR_LIT>".format(p[<NUM_LIT:1>]))<EOL><DEDENT>p[<NUM_LIT:0>] = p[<NUM_LIT:1>]<EOL>


p[<NUM_LIT:0>] = '<STR_LIT>'<EOL>


p[<NUM_LIT:0>] = ast.PredicatedExpression(p[<NUM_LIT:1>], p[<NUM_LIT:2>])<EOL>

PredicatedExpression : Expr PredicateList


p[<NUM_LIT:0>] = p[<NUM_LIT:1>]<EOL>p[<NUM_LIT:0>].append(p[<NUM_LIT:2>])<EOL>


p[<NUM_LIT:0>] = p[<NUM_LIT:2>]<EOL>

Predicate : OPEN_BRACKET Expr CLOSE_BRACKET


p[<NUM_LIT:0>] = ast.VariableReference(p[<NUM_LIT:2>])<EOL>


<EOL>if p[<NUM_LIT:1>] in ('<STR_LIT>', '<STR_LIT:text>'):<EOL><INDENT>p[<NUM_LIT:0>] = ast.NodeType(p[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>p[<NUM_LIT:0>] = ast.FunctionCall(p[<NUM_LIT:1>], p[<NUM_LIT:2>])<EOL><DEDENT>


p[<NUM_LIT:0>] = []<EOL>

FormalArguments : OPEN_PAREN CLOSE_PAREN


p[<NUM_LIT:0>] = p[<NUM_LIT:2>]<EOL>

FormalArguments : OPEN_PAREN ArgumentList CLOSE_PAREN


p[<NUM_LIT:0>] = [p[<NUM_LIT:1>]]<EOL>


if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj = next(obj.compute(ctx), False)<EOL><DEDENT>return to_boolean(obj)<EOL>

Handles LiteralObjects as well as computable arguments


if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj = next(obj.compute(ctx), False)<EOL><DEDENT>return to_number(obj)<EOL>


if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj = next(obj.compute(ctx), False)<EOL><DEDENT>return to_string(obj)<EOL>


if obj is None:<EOL><INDENT>item = ctx.item<EOL><DEDENT>elif hasattr(obj, '<STR_LIT>'):<EOL><INDENT>item = next(obj.compute(ctx), None)<EOL><DEDENT>else:<EOL><INDENT>item = obj<EOL><DEDENT>if isinstance(item, node):<EOL><INDENT>yield item.xml_name<EOL><DEDENT>else:<EOL><INDENT>yield '<STR_LIT>'<EOL><DEDENT>

Yields one string a node name or the empty string, operating on the first item in the provided obj, or the current item if obj is omitted
If this item is a node, yield its node name (generic identifier), otherwise yield ''
If obj is provided, but empty, yield ''


yield len(list(obj.compute(ctx)))<EOL>

Yields one number, count of items in the argument sequence


if seq is None:<EOL><INDENT>item = ctx.item<EOL><DEDENT>elif hasattr(seq, '<STR_LIT>'):<EOL><INDENT>item = next(seq.compute(ctx), '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>item = seq<EOL><DEDENT>yield next(to_string(item), '<STR_LIT>')<EOL>

Yields one string, derived from the argument literal (or the first item in the argument sequence, unless empty in which case yield '') as follows:

* If a node, yield its string-value
* If NaN, yield 'NaN'
* If +0 or -0, yield '0'
* If positive infinity, yield 'Infinity'
* If negative infinity, yield '-Infinity'
* If an integer, no decimal point and no leading zeros
* If a non-integer number, at least one digit before the decimal point and at least one digit after
* If boolean, either 'true' or 'false'


strings = flatten([ (s.compute(ctx) if callable(s) else s) for s in strings ])<EOL>strings = (next(string_arg(ctx, s), '<STR_LIT>') for s in strings)<EOL>yield '<STR_LIT>'.join(strings)<EOL>

Yields one string, concatenation of argument strings


@microxpath_function('<STR_LIT>')<EOL>def starts_with(ctx, full, part):

full = next(string_arg(ctx, full), '<STR_LIT>')<EOL>part = next(string_arg(ctx, part), '<STR_LIT>')<EOL>yield full.startswith(part)<EOL>

Yields one boolean, whether the first string starts with the second


@microxpath_function('<STR_LIT>')<EOL>def contains(ctx, full, part):

full = next(string_arg(ctx, full), '<STR_LIT>')<EOL>part = next(string_arg(ctx, part), '<STR_LIT>')<EOL>yield part in full<EOL>

Yields one boolean, whether the first string contains the second


@microxpath_function('<STR_LIT>')<EOL>def substring_before(ctx, full, part):

full = next(string_arg(ctx, full), '<STR_LIT>')<EOL>part = next(string_arg(ctx, part), '<STR_LIT>')<EOL>yield full.partition(part)[<NUM_LIT:0>]<EOL>


@microxpath_function('<STR_LIT>')<EOL>def substring_after(ctx, full, part):

full = next(string_arg(ctx, full), '<STR_LIT>')<EOL>part = next(string_arg(ctx, part), '<STR_LIT>')<EOL>yield full.partition(part)[-<NUM_LIT:1>]<EOL>


@microxpath_function('<STR_LIT>')<EOL>def substring(ctx, full, start, length):

full = next(string_arg(ctx, full), '<STR_LIT>')<EOL>start = int(next(to_number(start)))<EOL>length = int(next(to_number(length)))<EOL>yield full[start-<NUM_LIT:1>:start-<NUM_LIT:1>+length]<EOL>


if s is None:<EOL><INDENT>s = ctx.node<EOL><DEDENT>elif callable(s):<EOL><INDENT>s = next(s.compute(ctx), '<STR_LIT>')<EOL><DEDENT>yield len(s)<EOL>


<EOL>raise NotImplementedError<EOL>yield s<EOL>


@microxpath_function('<STR_LIT>')<EOL>def translate(ctx, s, subst):


@microxpath_function('<STR_LIT>')<EOL>def same_lang(ctx, seq, lang):


if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj = next(seq.compute(ctx), '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>obj = seq<EOL><DEDENT>yield next(to_boolean(obj), '<STR_LIT>')<EOL>

Yields one boolean, false if the argument sequence is empty, otherwise

* false if the first item is a boolean and false
* false if the first item is a number and positive or negative zero or NaN
* false if the first item is a string and ''
* true in all other cases


yield not next(boolean_arg(ctx, obj), False)<EOL>


yield True<EOL>


yield False<EOL>


if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj = next(seq.compute(ctx), '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>obj = seq<EOL><DEDENT>yield next(to_number(obj), '<STR_LIT>')<EOL>

Yields one float, derived from the first item in the argument sequence (unless empty in which case yield NaN) as follows:

* If string with optional whitespace followed by an optional minus sign followed by a Number followed by whitespace, converte to the IEEE 754 number that is nearest (according to the IEEE 754 round-to-nearest rule) to the mathematical value represented by the string; in case of any other string yield NaN
* If boolean true yield 1; if boolean false yield 0
* If a node convert to string as if by a call to string(); yield the same value as if passed that string argument to number()


@microxpath_function('<STR_LIT>')<EOL>def foreach_(ctx, seq, expr):

from . import context, parse as uxpathparse<EOL>if hasattr(seq, '<STR_LIT>'):<EOL><INDENT>seq = seq.compute(ctx)<EOL><DEDENT>expr = next(string_arg(ctx, expr), '<STR_LIT>')<EOL>pexpr = uxpathparse(expr)<EOL>for item in seq:<EOL><INDENT>innerctx = ctx.copy(item=item)<EOL>yield from pexpr.compute(innerctx)<EOL><DEDENT>

Yields the result of applying an expression to each item in the input sequence.

* seq: input sequence
* expr: expression to be converted to string, then dynamically evaluated for each item on the sequence to produce the result


@microxpath_function('<STR_LIT>')<EOL>def lookup_(ctx, tableid, key):

tableid = next(string_arg(ctx, tableid), '<STR_LIT>')<EOL>key = next(string_arg(ctx, key), '<STR_LIT>')<EOL>for item in seq:<EOL><INDENT>innerctx = ctx.copy(item=item)<EOL>yield from pexpr.compute(innerctx)<EOL><DEDENT>

Yields a sequence of a single value, the result of looking up a value from the tables provided in the context, or an empty sequence if lookup is unsuccessful

* tableid: id of the lookup table to use
* expr: expression to be converted to string, then dynamically evaluated for each item on the sequence to produce the result


<EOL>raise NotImplementedError<EOL>yield seq<EOL>

Yields one number, the sum, for each item in the argument sequence, of the result of converting as if by a call to number()


<EOL>raise NotImplementedError<EOL>yield num<EOL>

Yields one number, the largest (closest to positive infinity) number that is not greater than the argument and that is an integer.


Yields one number, the smallest (closest to negative infinity) number that is not less than the argument and that is an integer.


Yields one number, that which is closest to the argument and that is an integer. If there are two such numbers, then the one that is closest to positive infinity is returned. If the argument is NaN, then NaN is returned. If the argument is positive infinity, then positive infinity is returned. If the argument is negative infinity, then negative infinity is returned. If the argument is positive zero, then positive zero is returned. If the argument is negative zero, then negative zero is returned. If the argument is less than zero, but greater than or equal to -0.5, then negative zero is returned.


<EOL>return parser.parse(xpath, lexer=lexer)<EOL>

Parse an xpath.

>>> from amara3.uxml.uxpath import parse
>>> xp = parse('a/text()')


def qquery(xml_thing, xpath_thing, vars=None, funcs=None):

root = None<EOL>if isinstance(xml_thing, nodetype):<EOL><INDENT>root = xml_thing<EOL><DEDENT>elif isinstance(xml_thing, str):<EOL><INDENT>tb = tree.treebuilder()<EOL>root = tb.parse(xml_thing)<EOL><DEDENT>elif isinstance(xml_thing, bytes):<EOL><INDENT>tb = tree.treebuilder()<EOL>root = tb.parse(xml_thing.decode('<STR_LIT:utf-8>'))<EOL><DEDENT>if not root: return<EOL>if isinstance(xpath_thing, str):<EOL><INDENT>parsed_expr = parse(xpath_thing)<EOL><DEDENT>ctx = context(root, variables=vars, functions=funcs)<EOL>result = parsed_expr.compute(ctx)<EOL>yield from result<EOL>

Quick query. Convenience for using the MicroXPath engine.
Give it some XML and an expression and it will yield the results. No fuss.

xml_thing - bytes or string, or amara3.xml.tree node
xpath_thing - string or parsed XPath expression
vars - optional mapping of variables, name to value
funcs - optional mapping of functions, name to function object

>>> from amara3.uxml.uxpath import qquery
>>> results = qquery(b'123', 'a/text()'))
>>> next(results).xml_value
'1'
>>> next(results).xml_value
'3'


def __init__(self, item, pos=None, variables=None, functions=None, lookuptables=None, extras=None, parent=None, force_root=True):

self.item = item<EOL>if force_root and isinstance(item, nodetype):<EOL><INDENT>self.item = ast.root_node.get(item)<EOL><DEDENT>self.pos = pos<EOL>self.variables = variables or {}<EOL>self.functions = functions or BUILTIN_FUNCTIONS<EOL>self.extras = extras or {}<EOL>self.parent = parent or node.xml_parent<EOL>self.lookuptables = lookuptables or {}<EOL>

Note: No explicit context size. Will be dynamically computed if needed


t.value = t.value[<NUM_LIT:1>:-<NUM_LIT:1>]<EOL>return t<EOL>


t.value = float(t.value)<EOL>return t<EOL>


t.value = int(t.value)<EOL>return t<EOL>


for child in elem.xml_children:<EOL><INDENT>if isinstance(child, element):<EOL><INDENT>yield child<EOL>yield from descendants(child)<EOL><DEDENT><DEDENT>

Yields all the elements descendant of elem in document order


if isinstance(source, element):<EOL><INDENT>source = source.xml_children<EOL><DEDENT>return filter(lambda x: isinstance(x, element), source)<EOL>

Yields all the elements from the source
source - if an element, yields all child elements in order; if any other iterator yields the elements from that iterator


return filter(lambda x: x.xml_name == name, select_elements(source))<EOL>

Yields all the elements with the given name
source - if an element, starts with all child elements in order; can also be any other iterator
name - will yield only elements with this name


return filter(lambda x: pat.match(x.xml_name) is not None, select_elements(source))<EOL>

Yields elements from the source whose name matches the given regular expression pattern
source - if an element, starts with all child elements in order; can also be any other iterator
pat - re.pattern object


if isinstance(source, element):<EOL><INDENT>source = source.xml_children<EOL><DEDENT>return filter(lambda x: x.xml_value == val, source)<EOL>

Yields elements from the source with the given value (accumulated child text)
source - if an element, starts with all child elements in order; can also be any other iterator
val - string value to match


def select_attribute(source, name, val=None):

def check(x):<EOL><INDENT>if val is None:<EOL><INDENT>return name in x.xml_attributes<EOL><DEDENT>else:<EOL><INDENT>return name in x.xml_attributes and x.xml_attributes[name] == val<EOL><DEDENT><DEDENT>return filter(check, select_elements(source))<EOL>

Yields elements from the source having the given attrivute, optionally with the given attribute value
source - if an element, starts with all child elements in order; can also be any other iterator
name - attribute name to check
val - if None check only for the existence of the attribute, otherwise compare the given value as well


it = itertools.dropwhile(lambda x: x != elem, elem.xml_parent.xml_children)<EOL>next(it) <EOL>return it<EOL>

Yields elements and text which have the same parent as elem, but come afterward in document order


def select_pattern(node, pattern, state=None):

if state is None:<EOL><INDENT>state = _prep_pattern(pattern)<EOL><DEDENT>if isinstance(node, element):<EOL><INDENT>for child in node.xml_children:<EOL><INDENT>new_state = state(child)<EOL>if new_state == MATCHED_STATE:<EOL><INDENT>yield child<EOL><DEDENT>elif new_state is not None:<EOL><INDENT>yield from select_pattern(child, None, state=new_state)<EOL><DEDENT><DEDENT><DEDENT>return<EOL>

Yield descendant nodes matching the given pattern specification
pattern - tuple of steps, each of which matches an element by name, with "*" acting like a wildcard, descending the tree in tuple order
            sort of like a subset of XPath in Python tuple form
state - for internal use only

pattern examples:

("a", "b", "c") - all c elements whose parent is a b element whose parent is an a element whose parent is node
("*", "*") - any "grandchild" of node
("*", "*", "*") - any "great grandchild" of node
("**", "a") - any a descendant of node

>>> from amara3.uxml import tree
>>> from amara3.uxml.treeutil import *
>>>
>>> tb = tree.treebuilder()
>>> DOC = '<x>1</x><c><x>2</x><d><x>3</x></d></c><x>4</x><y>5</y>'
>>> root = tb.parse(DOC)
>>> results = [ e.xml_value for e in select_pattern(root, ('**', 'x')) ]
>>> results
['1', '2', '3', '4']


def make_pretty(elem, depth=<NUM_LIT:0>, indent='<STR_LIT:U+0020>'):

depth += <NUM_LIT:1><EOL>updated_child_list = []<EOL>updated_child_ix = <NUM_LIT:0><EOL>for child in elem.xml_children:<EOL><INDENT>if isinstance(child, element):<EOL><INDENT>if updated_child_ix % <NUM_LIT:2>:<EOL><INDENT>updated_child_list.append(child)<EOL>updated_child_ix += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>new_text = text('<STR_LIT:\n>' + indent*depth, elem)<EOL>updated_child_list.append(new_text)<EOL>updated_child_list.append(child)<EOL>updated_child_ix += <NUM_LIT:2><EOL><DEDENT>make_pretty(child, depth)<EOL><DEDENT>else:<EOL><INDENT>if child.xml_value.strip():<EOL><INDENT>updated_child_list.append(child)<EOL>updated_child_ix += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>new_text = text('<STR_LIT:\n>' + indent*depth, elem)<EOL>updated_child_list.append(new_text)<EOL>updated_child_ix += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>if not(updated_child_ix % <NUM_LIT:2>):<EOL><INDENT>new_text = text('<STR_LIT:\n>' + indent*(depth-<NUM_LIT:1>), elem)<EOL>updated_child_list.append(new_text)<EOL><DEDENT>elem.xml_children = updated_child_list<EOL>return elem<EOL>

Add text nodes as possible to all descendants of an element for spacing & indentation
to make the MicroXML as printed easier for people to read. Will not modify the
value of any text node which is not already entirely whitespace.

Warning: even though this operaton avoids molesting text nodes which already have
whitespace, it still makes changes which alter the text. Not all whitespace in XML is
ignorable. In XML cues from the DTD indicate which whitespace can be ignored.
No such cues are available for MicroXML, so use this function with care. That said,
in many real world applications of XML and MicroXML, this function causes no problems.

elem - target element whose descendant nodes are to be modified.
returns - the same element, which has been updated in place

>>> from amara3.uxml import tree
>>> from amara3.uxml.treeutil import *
>>> DOC = '<x>1</x><c><x>2</x><d><x>3</x></d></c><x>4</x><y>5</y>'
>>> tb = tree.treebuilder()
>>> root = tb.parse(DOC)
>>> len(root.xml_children)
4
>>> make_pretty(root)
<uxml.element (8763373718343) "a" with 9 children>
>>> len(root.xml_children)
9
>>> root.xml_encode()
'\n  \n    <x>1</x>\n  \n  <c>\n    <x>2</x>\n    <d>\n      <x>3</x>\n    </d>\n  </c>\n  <x>4</x>\n  <y>5</y>\n'


a_writer.start_element(elem.xml_name, attribs=elem.xml_attributes)<EOL>for node in elem.xml_children:<EOL><INDENT>if isinstance(node, tree.element):<EOL><INDENT>write(node, a_writer)<EOL><DEDENT>elif isinstance(node, tree.text):<EOL><INDENT>a_writer.text(node)<EOL><DEDENT><DEDENT>a_writer.end_element(elem.xml_name)<EOL>return<EOL>

Write a MicroXML element node (yes, even one representign a whole document)
elem - Amara MicroXML element node to be written out
writer - instance of amara3.uxml.writer to implement the writing process


def parse(source, prefixes=None, model=None, encoding=None, use_xhtml_ns=False):

def get_tree_instance(namespaceHTMLElements, use_xhtml_ns=use_xhtml_ns):<EOL><INDENT>return treebuilder(use_xhtml_ns)<EOL><DEDENT>parser = html5lib.HTMLParser(tree=get_tree_instance)<EOL>doc = parser.parse(source)<EOL>first_element = next((e for e in doc.root_nodes if isinstance(e, element)), None)<EOL>return first_element<EOL>

Parse an input source with HTML text into an Amara 3 tree

>>> from amara3.uxml import html5
>>> import urllib.request
>>> with urllib.request.urlopen('http://uche.ogbuji.net/') as response:
...     html5.parse(response)

#Warning: if you pass a string, you must make sure it's a byte string, not a Unicode object.  You might also want to wrap it with amara.lib.inputsource.text if it's not obviously XML or HTML (for example it could be confused with a file name)


doc = parse(source, encoding=encoding)<EOL>frag = doc.html.body<EOL>return frag<EOL>

Parse a fragment if markup in HTML mode, and return a bindery node

Warning: if you pass a string, you must make sure it's a byte string, not a Unicode object.  You might also want to wrap it with amara.lib.inputsource.text if it's not obviously XML or HTML (for example it could be confused with a file name)

from amara.lib import inputsource
from amara.bindery import html
doc = html.markup_fragment(inputsource.text('XXX
Spam!
Eggs!YYY'))

See also: http://wiki.xml3k.org/Amara2/Tagsoup


def insertText(self, data, insertBefore=None):

if insertBefore:<EOL><INDENT>self.insertBefore(tree.text(data), insertBefore)<EOL><DEDENT>else:<EOL><INDENT>self.xml_append(tree.text(data))<EOL><DEDENT>

Insert data as text in the current node, positioned before the
        start of node insertBefore or to the end of the node's text.


offset = self.xml_children.index(refNode)<EOL>self.xml_insert(node, offset)<EOL>

Insert node as a child of the current node, before refNode in the
        list of child nodes. Raises ValueError if refNode is not a child of
        the current node


raise NotImplementedError<EOL>

Return a shallow copy of the current node i.e. a node with the same
        name and attributes but with no parent or child nodes


return bool(self.xml_children)<EOL>

Return true if the node has children or text, false otherwise


attrs = self.xml_attributes.copy()<EOL>return element(self.xml_name, attrs=attrs)<EOL>