Retrieve entity path.

        Args:
            entity_id (str): The UUID of the requested entity.

        Returns:
            The path of the entity as a string::

                u'/12345/folder_1'

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_id)).return_body().get()["<STR_LIT>"]<EOL>

Retrieve entity Collab ID.

        Args:
            entity_id (str): The UUID of the requested entity.

        Returns:
            The id as interger of the Collebaration to which the entity belongs

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def get_entity_by_query(self, uuid=None, path=None, metadata=None):

if not (uuid or path or metadata):<EOL><INDENT>raise StorageArgumentException('<STR_LIT>')<EOL><DEDENT>if uuid and not is_valid_uuid(uuid):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(uuid))<EOL><DEDENT>params = locals().copy()<EOL>if metadata:<EOL><INDENT>if not isinstance(metadata, dict):<EOL><INDENT>raise StorageArgumentException('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>key, value = next(iter(metadata.items()))<EOL>params[key] = value<EOL>del params['<STR_LIT>']<EOL><DEDENT>params = self._prep_params(params)<EOL>return self._authenticated_request.to_endpoint('<STR_LIT>').with_params(params).return_body().get()<EOL>

Retrieve entity by query param which can be either uuid/path/metadata.

        Args:
            uuid (str): The UUID of the requested entity.
            path (str): The path of the requested entity.
            metadata (dict): A dictionary of one metadata {key: value} of the
                requested entitity.

        Returns:
            The details of the entity, if found::

                {
                    u'content_type': u'plain/text',
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-13T10:52:23.275087Z',
                    u'description': u'',
                    u'entity_type': u'file',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-13T10:52:23.275126Z',
                    u'name': u'myfile',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'e2c25c1b-f6a9-4cf6-b8d2-271e628a9a56'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def set_metadata(self, entity_type, entity_id, metadata):

if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>if not isinstance(metadata, dict):<EOL><INDENT>raise StorageArgumentException('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_type, entity_id)).with_json_body(metadata).return_body().post()<EOL>

Set metadata for an entity.

        Args:
            entity_type (str): Type of the entity. Admitted values: ['project',
                'folder', 'file'].
            entity_id (str): The UUID of the entity to be modified.
            metadata (dict): A dictionary of key/value pairs to be written as
                metadata.

        Warning:
            It will replace all existing metadata with the provided dictionary.

        Returns:
            A dictionary of the updated metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def get_metadata(self, entity_type, entity_id):

if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_type, entity_id)).return_body().get()<EOL>

Get metadata of an entity.

        Args:
            entity_type (str): Type of the entity. Admitted values: ['project',
                'folder', 'file'].
            entity_id (str): The UUID of the entity to be modified.

        Returns:
            A dictionary of the metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def update_metadata(self, entity_type, entity_id, metadata):

if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>if not isinstance(metadata, dict):<EOL><INDENT>raise StorageArgumentException('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_type, entity_id)).with_json_body(metadata).return_body().put()<EOL>

Update the metadata of an entity.

        Existing non-modified metadata will not be affected.

        Args:
            entity_type (str): Type of the entity. Admitted values: 'project',
                'folder', 'file'.
            entity_id (str): The UUID of the entity to be modified.
            metadata (dict): A dictionary of key/value pairs to be written as
                metadata.

        Returns:
            A dictionary of the updated object metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def delete_metadata(self, entity_type, entity_id, metadata_keys):

if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>if not isinstance(metadata_keys, list):<EOL><INDENT>raise StorageArgumentException('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_type, entity_id)).with_json_body({'<STR_LIT>': metadata_keys}).return_body().delete()<EOL>

Delete the selected metadata entries of an entity.

        Only deletes selected metadata keys, for a complete wipe, use set_metadata.

        Args:
            entity_type (str): Type of the entity. Admitted values: ['project',
                'folder', 'file'].
            entity_id (srt): The UUID of the entity to be modified.
            metadata_keys (lst): A list of metada keys to be deleted.

        Returns:
            A dictionary of the updated object metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def list_projects(self, hpc=None, access=None, name=None, collab_id=None,<EOL>page_size=DEFAULT_PAGE_SIZE, page=None, ordering=None):

return self._authenticated_request.to_endpoint('<STR_LIT>').with_params(self._prep_params(locals())).return_body().get()<EOL>

List all the projects the user have access to.

            This function does not retrieve all results, pages have
            to be manually retrieved by the caller.

        Args:
            hpc (bool): If 'true', the result will contain only the HPC projects
                (Unicore projects).
            access (str): If provided, the result will contain only projects
                where the user has the provided acccess.
                Admitted values: ['read', 'write'].
            name (str): Filter on the project name.
            collab_id (int): Filter on the collab id.
            page_size (int): Number of elements per page.
            page (int): Number of the page
            ordering (str): Indicate on which fields to sort the result.
                Prepend '-' to invert order. Multiple values can be provided.
                Ordering is supported on: ['name', 'created_on', 'modified_on'].
                Example: ordering='name,created_on'

        Returns:
            A dictionary of the results::

            {
                u'count': 256,
                u'next': u'http://link.to.next/page',
                u'previous': None,
                u'results': [{u'collab_id': 2079,
                    u'created_by': u'258666',
                    u'created_on': u'2017-02-23T15:09:27.626973Z',
                    u'description': u'',
                    u'entity_type': u'project',
                    u'modified_by': u'258666',
                    u'modified_on': u'2017-02-23T15:09:27.627025Z',
                    u'name': u'2079',
                    u'uuid': u'64a6ad2e-acd1-44a3-a4cd-6bd96e3da2b0'}]
            }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(project_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(project_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(project_id)).return_body().get()<EOL>

Get information on a given project

        Args:
            project_id (str): The UUID of the requested project.

        Returns:
            A dictionary describing the project::

            {
                u'collab_id': 2271,
                u'created_by': u'303447',
                u'created_on': u'2017-03-10T12:50:06.077891Z',
                u'description': u'',
                u'entity_type': u'project',
                u'modified_by': u'303447',
                u'modified_on': u'2017-03-10T12:50:06.077946Z',
                u'name': u'2271',
                u'uuid': u'3abd8742-d069-44cf-a66b-2370df74a682'
            }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def list_project_content(self, project_id, name=None, entity_type=None,<EOL>content_type=None, page_size=DEFAULT_PAGE_SIZE,<EOL>page=None, ordering=None):

if not is_valid_uuid(project_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(project_id))<EOL><DEDENT>params = self._prep_params(locals())<EOL>del params['<STR_LIT>']  <EOL>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(project_id)).with_params(params).return_body().get()<EOL>

List all files and folders (not recursively) contained in the project.

        This function does not retrieve all results, pages have
        to be manually retrieved by the caller.

        Args:
            project_id (str): The UUID of the requested project.
            name (str): Optional filter on entity name.
            entity_type (str): Optional filter on entity type.
                Admitted values: ['file', 'folder'].
            content_type (str): Optional filter on entity content type (only
                files are returned).
            page_size (int): Number of elements per page.
            page (int): Number of the page
            ordering (str): Indicate on which fields to sort the result.
                Prepend '-' to invert order. Multiple values can be provided.
                Ordering is supported on: ['name', 'created_on', 'modified_on'].
                Example: 'ordering=name,created_on'

        Returns:
            A dictionary of the results::

                {
                    u'count': 3,
                    u'next': u'http://link.to.next/page',
                    u'previous': None,
                    u'results': [{u'created_by': u'303447',
                        u'created_on': u'2017-03-10T17:41:31.618496Z',
                        u'description': u'',
                        u'entity_type': u'folder',
                        u'modified_by': u'303447',
                        u'modified_on': u'2017-03-10T17:41:31.618553Z',
                        u'name': u'folder_1',
                        u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                        u'uuid': u'eac11058-4ae0-4ea9-ada8-d3ea23887509'}]
                }

        Raises:
            StorageArgumentException: Ivalid parameters
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


return self._authenticated_request.to_endpoint('<STR_LIT>').with_json_body(self._prep_params(locals())).return_body().post()<EOL>

Create a new project.

        Args:
            collab_id (int): The id of the collab the project should be created in.

        Returns:
            A dictionary of details of the created project::

                {
                    u'collab_id': 12998,
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'project',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'12998',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


Delete a project. It will recursively delete all the content.

        Args:
            project (str): The UUID of the project to be deleted.

        Returns:
            None

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: 403
            StorageNotFoundException: 404
            HTTPError: other non-20x error codes


Create a new folder.

        Args:
            name (srt): The name of the folder.
            parent (str): The UUID of the parent entity. The parent must be a
                project or a folder.

        Returns:
            A dictionary of details of the created folder::

                {
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'folder',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'myfolder',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(folder):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(folder))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(folder)).return_body().get()<EOL>

Get information on a given folder.

        Args:
            folder (str): The UUID of the requested folder.

        Returns:
            A dictionary of the folder details if found::

                {
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'folder',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'myfolder',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def list_folder_content(self, folder, name=None, entity_type=None,<EOL>content_type=None, page_size=DEFAULT_PAGE_SIZE,<EOL>page=None, ordering=None):

if not is_valid_uuid(folder):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(folder))<EOL><DEDENT>params = self._prep_params(locals())<EOL>del params['<STR_LIT>']  <EOL>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(folder)).with_params(params).return_body().get()<EOL>

List files and folders (not recursively) contained in the folder.

        This function does not retrieve all results, pages have
        to be manually retrieved by the caller.

        Args:
            folder (str): The UUID of the requested folder.
            name (str): Optional filter on entity name.
            entity_type (str): Optional filter on entity type.
                Admitted values: ['file', 'folder'].
            content_type (str): Optional filter on entity content type (only
                files are returned).
            page_size (int): Number of elements per page.
            page (int): Number of the page.
            ordering (str): Indicate on which fields to sort the result. Prepend
                '-' to invert order. Multiple values can be provided.
                Ordering is supported on: ['name', 'created_on', 'modified_on'].
                Example: 'ordering=name,created_on'

        Returns:
            A dictionary of the results::

                {
                u'count': 1,
                u'next': None,
                u'previous': None,
                u'results': [{u'content_type': u'plain/text',
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-13T10:17:01.688472Z',
                    u'description': u'',
                    u'entity_type': u'file',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-13T10:17:01.688632Z',
                    u'name': u'file_1',
                    u'parent': u'eac11058-4ae0-4ea9-ada8-d3ea23887509',
                    u'uuid': u'0e17eaac-cb00-4336-b9d7-657026844281'}]
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(folder):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(folder))<EOL><DEDENT>self._authenticated_request.to_endpoint('<STR_LIT>'.format(folder)).delete()<EOL>

Delete a folder. It will recursively delete all the content.

        Args:
            folder_id (str): The UUID of the folder to be deleted.

        Returns:
            None

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: 403
            StorageNotFoundException: 404
            HTTPError: other non-20x error codes


def create_file(self, name, content_type, parent):

Create a new file.

        Args:
            name (str): The name of the file.
            content_type (str): the content type of the file. E.g. "plain/text"
            parent (str): The UUID of the parent entity. The parent must be a
                project or a folder.

        Returns:
            A dictionary describing the created file::

                {
                    u'content_type': u'plain/text',
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-13T10:52:23.275087Z',
                    u'description': u'',
                    u'entity_type': u'file',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-13T10:52:23.275126Z',
                    u'name': u'myfile',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'e2c25c1b-f6a9-4cf6-b8d2-271e628a9a56'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: 403
            StorageNotFoundException: 404
            HTTPError: other non-20x error codes


if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).return_body().get()<EOL>

Get information on a given file.

        Args:
            file_id (str): The UUID of the requested file.

        Returns:
            A dictionary of the file details if found::

                {
                    u'content_type': u'plain/text',
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-13T10:52:23.275087Z',
                    u'description': u'',
                    u'entity_type': u'file',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-13T10:52:23.275126Z',
                    u'name': u'myfile',
                    u'parent': u'3abd8742-d069-44cf-a66b-2370df74a682',
                    u'uuid': u'e2c25c1b-f6a9-4cf6-b8d2-271e628a9a56'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def upload_file_content(self, file_id, etag=None, source=None, content=None):

if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>if not (source or content) or (source and content):<EOL><INDENT>raise StorageArgumentException('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>resp = self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).with_body(content or open(source, '<STR_LIT:rb>')).with_headers({'<STR_LIT>': etag} if etag else {}).post()<EOL>if '<STR_LIT>' not in resp.headers:<EOL><INDENT>raise StorageException('<STR_LIT>')<EOL><DEDENT>return resp.headers['<STR_LIT>']<EOL>

Upload a file content. The file entity must already exist.

        If an ETag is provided the file stored on the server is verified
        against it. If it does not match, StorageException is raised.
        This means the client needs to update its knowledge of the resource
        before attempting to update again. This can be used for optimistic
        concurrency control.

        Args:
            file_id (str): The UUID of the file whose content is written.
            etag (str): The etag to match the contents against.
            source (str): The path of the local file whose content to be uploaded.
            content (str): A string of the content to be uploaded.

        Note:
            ETags should be enclosed in double quotes::

                my_etag = '"71e1ed9ee52e565a56aec66bc648a32c"'

        Returns:
            The ETag of the file upload::

                '"71e1ed9ee52e565a56aec66bc648a32c"'

        Raises:
            IOError: The source cannot be opened.
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def copy_file_content(self, file_id, source_file):

if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>if not is_valid_uuid(source_file):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(source_file))<EOL><DEDENT>self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).with_headers({'<STR_LIT>': source_file}).put()<EOL>

Copy file content from source file to target file.

        Args:
            file_id (str): The UUID of the file whose content is written.
            source_file (str): The UUID of the file whose content is copied.

        Returns:
            None

        Raises:
        StorageArgumentException: Invalid arguments
        StorageForbiddenException: Server response code 403
        StorageNotFoundException: Server response code 404
        StorageException: other 400-600 error codes


def download_file_content(self, file_id, etag=None):

if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>headers = {'<STR_LIT>': '<STR_LIT>'}<EOL>if etag:<EOL><INDENT>headers['<STR_LIT>'] = etag<EOL><DEDENT>resp = self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).with_headers(headers).get()<EOL>if resp.status_code == <NUM_LIT>:<EOL><INDENT>return (None, None)<EOL><DEDENT>if '<STR_LIT>' not in resp.headers:<EOL><INDENT>raise StorageException('<STR_LIT>')<EOL><DEDENT>return (resp.headers['<STR_LIT>'], resp.content)<EOL>

Download file content.

        Args:
            file_id (str): The UUID of the file whose content is requested
            etag (str): If the content is not changed since the provided ETag,
                the content won't be downloaded. If the content is changed, it
                will be downloaded and returned with its new ETag.

        Note:
            ETags should be enclosed in double quotes::

                my_etag = '"71e1ed9ee52e565a56aec66bc648a32c"'

        Returns:
            A tuple of ETag and content (etag, content) if the content was
            retrieved. If an etag was provided, and content didn't change
            returns (None, None)::

                ('"71e1ed9ee52e565a56aec66bc648a32c"', 'Hello world!')

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).return_body().get()['<STR_LIT>']<EOL>

Get a signed unauthenticated URL.

        It can be used to download the file content without the need for a
        token. The signed URL expires after 5 seconds.

        Args:
            file_id (str): The UUID of the file to get the link for.

        Returns:
            The signed url as a string

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


Delete a file.

        Args:
            file_id (str): The UUID of the file to delete.

        Returns:
            None

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


return self._request.to_endpoint(signed_url).stream_response().get()<EOL>

Downloads a file with its signed url.

        Args:
            signed_url (str): The signed url of the file to download.

        Returns:
            The streamed response which is used to retrieve the file content

            with open(target_file, "wb") as output:
                for chunk in response.iter_content(chunk_size=1024):
                    output.write(chunk)

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


prevdir = os.getcwd()<EOL>os.chdir(newdir)<EOL>try:<EOL><INDENT>yield<EOL><DEDENT>finally:<EOL><INDENT>os.chdir(prevdir)<EOL><DEDENT>


try:<EOL><INDENT>stdout = completed_cmd.stdout.encode('<STR_LIT:utf-8>').decode()<EOL><DEDENT>except AttributeError:<EOL><INDENT>try:<EOL><INDENT>stdout = str(bytes(completed_cmd.stdout), '<STR_LIT>').strip()<EOL><DEDENT>except AttributeError:<EOL><INDENT>stdout = str(bytes(completed_cmd.stdout).decode('<STR_LIT:utf-8>')).strip()<EOL><DEDENT><DEDENT>try:<EOL><INDENT>stderr = completed_cmd.stderr.encode('<STR_LIT:utf-8>').decode()<EOL><DEDENT>except AttributeError:<EOL><INDENT>try:<EOL><INDENT>stderr = str(bytes(completed_cmd.stderr), '<STR_LIT>').strip()<EOL><DEDENT>except AttributeError:<EOL><INDENT>stderr = str(bytes(completed_cmd.stderr).decode('<STR_LIT:utf-8>')).strip()<EOL><DEDENT><DEDENT>return ParsedCompletedCommand(<EOL>completed_cmd.returncode,<EOL>completed_cmd.args,<EOL>stdout,<EOL>stderr<EOL>)<EOL>


def run_command_under_r_root(self, cmd, catched=True):

RPATH = self.path<EOL>with self.cd(newdir=RPATH):<EOL><INDENT>if catched:<EOL><INDENT>process = sp.run(cmd, stdout=sp.PIPE, stderr=sp.PIPE)<EOL><DEDENT>else:<EOL><INDENT>process = sp.run(cmd)<EOL><DEDENT>return process<EOL><DEDENT>


rprocess = OrderedDict()<EOL>commands = OrderedDict([<EOL>(self.file, ['<STR_LIT>', self.file] + self.cmd),<EOL>])<EOL>for cmd_name, cmd in commands.items():<EOL><INDENT>rprocess[cmd_name] = self.run_command_under_r_root(cmd)<EOL><DEDENT>return self.decode_cmd_out(completed_cmd=rprocess[self.file])<EOL>


def __init__(self, method='<STR_LIT>', enabled=True, experiment=None):

self.method = method<EOL>self.enabled = enabled<EOL>self.experiment = experiment<EOL>self.seen_params = {}<EOL>self.current_iteration = <NUM_LIT:0><EOL>self.trials = None<EOL>self.nb_iterations = None<EOL>self.params = []<EOL>

:param method: 'grid_search', 'random_search'
:param enabled:


def optimize(self, fx, nb_iterations=None):

self.nb_iterations = nb_iterations<EOL>result = fx(self)<EOL>if self.experiment is not None:<EOL><INDENT>result['<STR_LIT>'] = self.current_iteration<EOL>self.experiment.log(result)<EOL><DEDENT>self.current_iteration += <NUM_LIT:1><EOL>self.__generate_trials()<EOL>for i in range(<NUM_LIT:1>, len(self.trials)):<EOL><INDENT>result = fx(self)<EOL>result['<STR_LIT>'] = self.current_iteration<EOL>if self.experiment is not None:<EOL><INDENT>self.experiment.log(result)<EOL><DEDENT>self.current_iteration += <NUM_LIT:1><EOL><DEDENT>

Primary entry point into the optimization
:param fx:
:param nb_iterations:
:return:


flat_params = self.__flatten_params(self.params)<EOL>if self.method == '<STR_LIT>':<EOL><INDENT>self.trials = list(itertools.product(*flat_params))<EOL>if self.nb_iterations is not None:<EOL><INDENT>self.trials = self.trials[<NUM_LIT:0>: self.nb_iterations]<EOL><DEDENT><DEDENT>if self.method == '<STR_LIT>':<EOL><INDENT>self.trials = self.__generate_random_search_trials(flat_params)<EOL><DEDENT>

Generates the parameter combinations for each requested trial
:return:


flat_params = []<EOL>for i, param in enumerate(params):<EOL><INDENT>param_groups = []<EOL>for val in param['<STR_LIT>']:<EOL><INDENT>param_groups.append({'<STR_LIT>': i, '<STR_LIT>': val})<EOL><DEDENT>flat_params.append(param_groups)<EOL><DEDENT>return flat_params<EOL>

Turns a list of parameters with values into a flat tuple list of lists
so we can permute
:param params:
:return:


<EOL>if save_dir is not None:<EOL><INDENT>global _ROOT<EOL>_ROOT = save_dir<EOL><DEDENT>self.metrics = []<EOL>self.tags = {}<EOL>self.name = name<EOL>self.debug = debug<EOL>self.version = version<EOL>self.autosave = autosave<EOL>self.description = description<EOL>self.create_git_tag = create_git_tag<EOL>self.exp_hash = '<STR_LIT>'.format(self.name, version)<EOL>self.created_at = str(datetime.utcnow())<EOL>if version is None:<EOL><INDENT>old_version = self.__get_last_experiment_version()<EOL>self.exp_hash = '<STR_LIT>'.format(self.name, old_version + <NUM_LIT:1>)<EOL>self.version = old_version + <NUM_LIT:1><EOL><DEDENT>if not debug:<EOL><INDENT>self.__init_cache_file_if_needed()<EOL>if self.version is not None:<EOL><INDENT>if not os.path.exists(self.__get_log_name()):<EOL><INDENT>self.__create_exp_file(self.version)<EOL>self.save()<EOL><DEDENT>else:<EOL><INDENT>self.__load()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>old_version = self.__get_last_experiment_version()<EOL>self.version = old_version<EOL>self.__create_exp_file(self.version + <NUM_LIT:1>)<EOL>self.save()<EOL><DEDENT>if self.create_git_tag == True:<EOL><INDENT>desc = description if description is not None else '<STR_LIT>'<EOL>tag_msg = '<STR_LIT>'.format(self.name, desc)<EOL>cmd = '<STR_LIT>'.format(self.exp_hash, tag_msg)<EOL>os.system(cmd)<EOL>print('<STR_LIT>', '<STR_LIT>'.format(self.exp_hash))<EOL><DEDENT><DEDENT>

A new Experiment object defaults to 'default' unless a specific name is provided
If a known name is already provided, then the file version is changed
:param name:
:param debug:


meta = hypo.get_current_trial_meta()<EOL>for tag in meta:<EOL><INDENT>self.tag(tag)<EOL><DEDENT>

Transfers meta data about all the params from the
hyperoptimizer to the log
:param hypo:
:return:


exp_cache_file = self.get_data_path(self.name, self.version)<EOL>if not os.path.isdir(exp_cache_file):<EOL><INDENT>os.makedirs(exp_cache_file)<EOL><DEDENT>

Inits a file that we log historical experiments
:return:


exp_cache_file = self.get_data_path(self.name, self.version)<EOL>path = '<STR_LIT>'.format(exp_cache_file)<EOL>open(path, '<STR_LIT:w>').close()<EOL>self.version = version<EOL>os.mkdir(self.get_media_path(self.name, self.version))<EOL>

Recreates the old file with this exp and version
:param version:
:return:


if self.debug: return<EOL>for k, v in tag_dict.items():<EOL><INDENT>self.tags[k] = v<EOL><DEDENT>if self.autosave == True:<EOL><INDENT>self.save()<EOL><DEDENT>

Adds a tag to the experiment.
Tags are metadata for the exp.

>> e.tag({"model": "Convnet A"})

:param key:
:param val:
:return:


if self.debug: return<EOL>if '<STR_LIT>' not in metrics_dict:<EOL><INDENT>metrics_dict['<STR_LIT>'] = str(datetime.utcnow())<EOL><DEDENT>self.__convert_numpy_types(metrics_dict)<EOL>self.metrics.append(metrics_dict)<EOL>if self.autosave == True:<EOL><INDENT>self.save()<EOL><DEDENT>

Adds a json dict of metrics.

>> e.log({"loss": 23, "coeff_a": 0.2})

:param metrics_dict:
:return:


if self.debug: return<EOL>self.__save_images(self.metrics)<EOL>metrics_file_path = self.get_data_path(self.name, self.version) + '<STR_LIT>'<EOL>meta_tags_path = self.get_data_path(self.name, self.version) + '<STR_LIT>'<EOL>obj = {<EOL>'<STR_LIT:name>': self.name,<EOL>'<STR_LIT:version>': self.version,<EOL>'<STR_LIT>': meta_tags_path,<EOL>'<STR_LIT>': metrics_file_path,<EOL>'<STR_LIT>': self.autosave,<EOL>'<STR_LIT:description>': self.description,<EOL>'<STR_LIT>': self.created_at,<EOL>'<STR_LIT>': self.exp_hash<EOL>}<EOL>with open(self.__get_log_name(), '<STR_LIT:w>') as file:<EOL><INDENT>json.dump(obj, file, ensure_ascii=False)<EOL><DEDENT>df = pd.DataFrame({'<STR_LIT:key>': list(self.tags.keys()), '<STR_LIT:value>': list(self.tags.values())})<EOL>df.to_csv(meta_tags_path, index=False)<EOL>df = pd.DataFrame(self.metrics)<EOL>df.to_csv(metrics_file_path, index=False)<EOL>

Saves current experiment progress
:return:


<EOL>for i, metric in enumerate(metrics):<EOL><INDENT>for k, v in metric.items():<EOL><INDENT>img_extension = None<EOL>img_extension = '<STR_LIT>' if '<STR_LIT>' in k else img_extension<EOL>img_extension = '<STR_LIT>' if '<STR_LIT>' in k else img_extension<EOL>img_extension = '<STR_LIT>' if '<STR_LIT>' in k else img_extension<EOL>if img_extension is not None:<EOL><INDENT>img_name = '<STR_LIT:_>'.join(k.split('<STR_LIT:_>')[<NUM_LIT:1>:])<EOL>save_path = self.get_media_path(self.name, self.version)<EOL>save_path = '<STR_LIT>'.format(save_path, img_name, i, img_extension)<EOL>if type(metric[k]) is not str:<EOL><INDENT>imwrite(save_path, metric[k])<EOL><DEDENT>metric[k] = save_path<EOL><DEDENT><DEDENT><DEDENT>

Save tags that have a png_ prefix (as images)
and replace the meta tag with the file name
:param metrics:
:return:


def get_data_path(self, exp_name, exp_version):

return os.path.join(_ROOT, '<STR_LIT>', exp_name, '<STR_LIT>'.format(exp_version))<EOL>

Returns the path to the local package cache
:param path:
:return:


def get_media_path(self, exp_name, exp_version):

return os.path.join(self.get_data_path(exp_name, exp_version), '<STR_LIT>')<EOL>


def __init__(self, strategy='<STR_LIT>', **kwargs):

ArgumentParser.__init__(self, **kwargs)<EOL>self.strategy = strategy<EOL>self.trials = []<EOL>self.parsed_args = None<EOL>self.opt_args = {}<EOL>self.json_config_arg_name = None<EOL>self.pool = None<EOL>

:param strategy: 'grid_search', 'random_search'
:param enabled:
:param experiment:
:param kwargs:


self.trials = strategies.generate_trials(<EOL>strategy=self.strategy,<EOL>flat_params=self.__flatten_params(self.opt_args),<EOL>nb_trials=nb_trials,<EOL>)<EOL>self.trials = [(self.__namespace_from_trial(x), train_function) for x in self.trials]<EOL>if self.pool is None:<EOL><INDENT>gpu_q = Queue()<EOL>for gpu_id in gpu_ids:<EOL><INDENT>gpu_q.put(gpu_id)<EOL><DEDENT>def init(local_gpu_q):<EOL><INDENT>global g_gpu_id_q<EOL>g_gpu_id_q = local_gpu_q<EOL><DEDENT>self.pool = Pool(processes=nb_workers, initializer=init, initargs=(gpu_q,))<EOL><DEDENT>results = self.pool.map(optimize_parallel_gpu_private, self.trials)<EOL>return results<EOL>

Runs optimization across gpus with cuda drivers
:param train_function:
:param nb_trials:
:param gpu_ids: List of strings like: ['0', '1, 3']
:param nb_workers:
:return:


self.trials = trials<EOL>self.trials = [(x, train_function) for x in self.trials]<EOL>if self.pool is None:<EOL><INDENT>gpu_q = Queue()<EOL>for gpu_id in gpu_ids:<EOL><INDENT>gpu_q.put(gpu_id)<EOL><DEDENT>def init(local_gpu_q):<EOL><INDENT>global g_gpu_id_q<EOL>g_gpu_id_q = local_gpu_q<EOL><DEDENT>self.pool = Pool(processes=nb_workers, initializer=init, initargs=(gpu_q,))<EOL><DEDENT>results = self.pool.map(optimize_parallel_gpu_private, self.trials)<EOL>return results<EOL>


self.trials = strategies.generate_trials(<EOL>strategy=self.strategy,<EOL>flat_params=self.__flatten_params(self.opt_args),<EOL>nb_trials=nb_trials<EOL>)<EOL>self.trials = [(self.__namespace_from_trial(x), train_function) for x in self.trials]<EOL>if self.pool is None:<EOL><INDENT>self.pool = Pool(processes=nb_workers)<EOL><DEDENT>results = self.pool.map(optimize_parallel_cpu_private, self.trials)<EOL>return results<EOL>

Runs optimization across n cpus
:param train_function:
:param nb_trials:
:param nb_workers:
:return:


flat_params = []<EOL>for i, (opt_name, opt_arg) in enumerate(params.items()):<EOL><INDENT>if opt_arg.tunable:<EOL><INDENT>clean_name = opt_name.strip('<STR_LIT:->')<EOL>clean_name = re.sub('<STR_LIT:->', '<STR_LIT:_>', clean_name)<EOL>param_groups = []<EOL>for val in opt_arg.opt_values:<EOL><INDENT>param_groups.append({'<STR_LIT>': i, '<STR_LIT>': val, '<STR_LIT:name>': clean_name})<EOL><DEDENT>flat_params.append(param_groups)<EOL><DEDENT><DEDENT>return flat_params<EOL>


def generate_trials(strategy, flat_params, nb_trials=None):

if strategy == '<STR_LIT>':<EOL><INDENT>trials = generate_grid_search_trials(flat_params, nb_trials)<EOL>return trials<EOL><DEDENT>elif strategy == '<STR_LIT>':<EOL><INDENT>trials = generate_random_search_trials(flat_params, nb_trials)<EOL>return trials<EOL><DEDENT>else:<EOL><INDENT>raise ValueError(<EOL>('<STR_LIT>'<EOL>'<STR_LIT>').format(strategy))<EOL><DEDENT>

r"""Generates the parameter combinations to search.

    Two search strategies are implemented:
    1. `grid_search`: creates a search space that consists of the
        product of all flat_params. If `nb_trials` is specified
        the first `nb_trials` combinations are searched.
    2. `random_search`: Creates random combinations of the
        hyperparameters. Can be used for a more efficient search.
        See (Bergstra and Bengio, 2012) for more details.

    :param strategy: The hyperparameter search to strategy. Can be
        one of: {`grid_search`, `random`}.
    :param flat_params: The hyperparameter arguments to iterate over.
    :param nb_trials: The number of hyperparameter combinations to try.
    Generates the parameter combinations for each requested trial
    :param strategy:
    :param flat_params:
    :param nb_trials: The number of trials to un.
    :return:


trials = list(itertools.product(*flat_params))<EOL>if nb_trials:<EOL><INDENT>trials = trials[<NUM_LIT:0>:nb_trials]<EOL><DEDENT>return trials<EOL>

Standard grid search. Takes the product of `flat_params`
to generate the search space.

:param params: The hyperparameters options to search.
:param nb_trials: Returns the first `nb_trials` from the
    combinations space. If this is None, all combinations
    are returned.
:return: A dict containing the hyperparameters.


if nb_trials is None:<EOL><INDENT>raise TypeError(<EOL>'<STR_LIT>')<EOL><DEDENT>results = []<EOL>seen_trials = set()<EOL>potential_trials = <NUM_LIT:1><EOL>for param in params:<EOL><INDENT>random.shuffle(param)<EOL>potential_trials *= len(param)<EOL><DEDENT>max_iters = min(potential_trials, nb_trials)<EOL>while len(results) < max_iters:<EOL><INDENT>trial = []<EOL>for param in params:<EOL><INDENT>sampled_param = random.sample(param, <NUM_LIT:1>)[<NUM_LIT:0>]<EOL>trial.append(sampled_param)<EOL><DEDENT>trial_str = json.dumps(trial)<EOL>if trial_str not in seen_trials:<EOL><INDENT>seen_trials.add(trial_str)<EOL>results.append(trial)<EOL><DEDENT><DEDENT>return results<EOL>

Generates random combination of hyperparameters to try.
See (Bergstra and Bengio, 2012) for more details.

:param params: The hyperparameters options to search.
:param nb_trials: The number of trials to run.
:return: A dict containing the hyperparameters.


self.job_name = job_name<EOL>self.job_display_name = job_display_name<EOL>self.on_gpu = on_gpu<EOL>self.__layout_logging_dir()<EOL>if self.is_from_slurm_object:<EOL><INDENT>self.__run_experiment(train_function)<EOL><DEDENT>else:<EOL><INDENT>trials = self.hyperparam_optimizer.generate_trials(nb_trials)<EOL>next_test_tube_version = self.__get_max_test_tube_version(self.log_path)<EOL>for i, trial_params in enumerate(trials):<EOL><INDENT>exp_i = i + next_test_tube_version<EOL>self.schedule_experiment(trial_params, exp_i)<EOL><DEDENT><DEDENT>

Runs optimization on the attached cluster
:param train_function:
:param nb_trials:
:param job_name:
:return:


def __call_old_slurm_cmd(self, original_slurm_cmd_script_path, exp_i, copy_current=True):

<EOL>script_path = original_slurm_cmd_script_path.split('<STR_LIT>')[<NUM_LIT:0>] + '<STR_LIT>'<EOL>timestamp = datetime.datetime.now().strftime("<STR_LIT>")<EOL>timestamp = '<STR_LIT>'.format(exp_i, timestamp)<EOL>new_slurm_cmd_script_path = os.path.join(script_path, '<STR_LIT>'.format(timestamp))<EOL>copyfile(original_slurm_cmd_script_path, new_slurm_cmd_script_path)<EOL>old_file = open(original_slurm_cmd_script_path)<EOL>lines = old_file.read().split('<STR_LIT:\n>')<EOL>last_line = lines[-<NUM_LIT:1>]<EOL>lines = [line + '<STR_LIT:\n>' for line in lines]<EOL>lines[-<NUM_LIT:1>] = last_line<EOL>old_file.close()<EOL>if not HyperOptArgumentParser.SLURM_LOAD_CMD in lines[-<NUM_LIT:1>]:<EOL><INDENT>last_line = lines[-<NUM_LIT:1>]<EOL>last_line = '<STR_LIT>'.format(last_line, HyperOptArgumentParser.SLURM_LOAD_CMD)<EOL>lines[-<NUM_LIT:1>] = last_line<EOL>open(new_slurm_cmd_script_path, '<STR_LIT:w>').writelines(lines)<EOL><DEDENT>print('<STR_LIT>')<EOL>result = call('<STR_LIT>'.format(AbstractCluster.RUN_CMD, new_slurm_cmd_script_path), shell=True)<EOL>if result == <NUM_LIT:0>:<EOL><INDENT>print('<STR_LIT>', new_slurm_cmd_script_path)<EOL><DEDENT>else:<EOL><INDENT>print('<STR_LIT>')<EOL><DEDENT>

Copies old slurm script into a new one and adds a load flag in case it wasn't there.
Then schedules the script again, but this time with the load flag which will signal the program
to load the model so it can continue training.

:param original_slurm_cmd_script_path:
:param exp_i:
:param copy_current:
:return:


<EOL>if self.test_tube_exp_name is not None:<EOL><INDENT>slurm_out_path = os.path.join(self.log_path, self.test_tube_exp_name)<EOL><DEDENT>else:<EOL><INDENT>slurm_out_path = os.path.join(self.log_path, self.job_name)<EOL><DEDENT>self.log_path = slurm_out_path<EOL>if not os.path.exists(slurm_out_path):<EOL><INDENT>os.makedirs(slurm_out_path)<EOL><DEDENT>if self.enable_log_err:<EOL><INDENT>err_path = os.path.join(slurm_out_path, '<STR_LIT>')<EOL>if not os.path.exists(err_path):<EOL><INDENT>os.makedirs(err_path)<EOL><DEDENT>self.err_log_path = err_path<EOL><DEDENT>if self.enable_log_out:<EOL><INDENT>out_path = os.path.join(slurm_out_path, '<STR_LIT>')<EOL>if not os.path.exists(out_path):<EOL><INDENT>os.makedirs(out_path)<EOL><DEDENT>self.out_log_path = out_path<EOL><DEDENT>self.slurm_files_log_path = os.path.join(slurm_out_path, '<STR_LIT>')<EOL>if not os.path.exists(self.slurm_files_log_path):<EOL><INDENT>os.makedirs(self.slurm_files_log_path)<EOL><DEDENT>

Generates dir structure for logging errors and outputs
:return:


params = []<EOL>for k in trial.__dict__:<EOL><INDENT>v = trial.__dict__[k]<EOL>if v is None or v == False:<EOL><INDENT>continue<EOL><DEDENT>if self.__should_escape(v):<EOL><INDENT>cmd = '<STR_LIT>'.format(k, v)<EOL><DEDENT>else:<EOL><INDENT>cmd = '<STR_LIT>'.format(k, v)<EOL><DEDENT>params.append(cmd)<EOL><DEDENT>params.append('<STR_LIT>'.format(HyperOptArgumentParser.TRIGGER_CMD))<EOL>full_cmd = '<STR_LIT:U+0020>'.join(params)<EOL>return full_cmd<EOL>

Turns hopt trial into script params
:param trial:
:return:


<EOL>exp = Experiment(<EOL>name=hparams.test_tube_exp_name,<EOL>save_dir=hparams.log_path,<EOL>autosave=False,<EOL>)<EOL>exp.argparse(hparams)<EOL>x = torch.rand((<NUM_LIT:1>, hparams.x_val))<EOL>for train_step in range(<NUM_LIT:0>, <NUM_LIT:100>):<EOL><INDENT>y = torch.rand((hparams.x_val, <NUM_LIT:1>))<EOL>out = x.mm(y)<EOL>exp.log({'<STR_LIT>': out.item()})<EOL><DEDENT>exp.save()<EOL>

Train your awesome model.

    :param hparams: The arguments to run the model with.


<EOL>exp = Experiment(<EOL>name=hparams.test_tube_exp_name,<EOL>save_dir=hparams.log_path,<EOL>version=hparams.hpc_exp_number,<EOL>autosave=False,<EOL>)<EOL>exp.argparse(hparams)<EOL>x = hparams.x_val<EOL>for train_step in range(<NUM_LIT:0>, <NUM_LIT:100>):<EOL><INDENT>y = hparams.y_val<EOL>out = x * y<EOL>exp.log({'<STR_LIT>': out.item()})  <EOL><DEDENT>exp.save()<EOL>


def run_setup(script_name, script_args=None, stop_after="<STR_LIT>"):

if stop_after not in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>raise ValueError("<STR_LIT>" % stop_after)<EOL><DEDENT>core._setup_stop_after = stop_after<EOL>save_argv = sys.argv<EOL>glocals = copy(globals())<EOL>glocals['<STR_LIT>'] = script_name<EOL>glocals['<STR_LIT>'] = "<STR_LIT:__main__>"<EOL>try:<EOL><INDENT>try:<EOL><INDENT>sys.argv[<NUM_LIT:0>] = script_name<EOL>if script_args is not None:<EOL><INDENT>sys.argv[<NUM_LIT:1>:] = script_args<EOL><DEDENT>f = open(script_name)<EOL>try:<EOL><INDENT>exec(f.read(), glocals, glocals)<EOL><DEDENT>finally:<EOL><INDENT>f.close()<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>sys.argv = save_argv<EOL>core._setup_stop_after = None<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>logging.warn("<STR_LIT>", exc_info=True)<EOL><DEDENT>if core._setup_distribution is None:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>script_name)<EOL><DEDENT>return core._setup_distribution<EOL>

Run a setup script in a somewhat controlled environment, and
    return the Distribution instance that drives things.  This is useful
    if you need to find out the distribution meta-data (passed as
    keyword args from 'script' to 'setup()', or the contents of the
    config files or command-line.

    'script_name' is a file that will be run with 'execfile()';
    'sys.argv[0]' will be replaced with 'script' for the duration of the
    call.  'script_args' is a list of strings; if supplied,
    'sys.argv[1:]' will be replaced by 'script_args' for the duration of
    the call.

    'stop_after' tells 'setup()' when to stop processing; possible
    values:
      init
        stop after the Distribution instance has been created and
        populated with the keyword arguments to 'setup()'
      config
        stop after config files have been parsed (and their data
        stored in the Distribution instance)
      commandline
        stop after the command-line ('sys.argv[1:]' or 'script_args')
        have been parsed (and the data stored in the Distribution)
      run [default]
        stop after all commands have been run (the same as if 'setup()'
        had been called in the usual way

    Returns the Distribution instance, which provides all information
    used to drive the Distutils.


<EOL>with FakeContext(path):<EOL><INDENT>with SetupMonkey() as sm:<EOL><INDENT>try:<EOL><INDENT>distro = run_setup('<STR_LIT>', stop_after='<STR_LIT>')<EOL>metadata = {'<STR_LIT>': sm.used_setuptools}<EOL>for k, v in distro.metadata.__dict__.items():<EOL><INDENT>if k[<NUM_LIT:0>] == '<STR_LIT:_>' or not v:<EOL><INDENT>continue<EOL><DEDENT>if all(not x for x in v):<EOL><INDENT>continue<EOL><DEDENT>metadata[k] = v<EOL><DEDENT>if sm.used_setuptools:<EOL><INDENT>for extras in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>v = getattr(distro, extras, None)<EOL>if v is not None and v not in ([], {}):<EOL><INDENT>metadata[extras] = v<EOL><DEDENT><DEDENT><DEDENT><DEDENT>except ImportError as e:<EOL><INDENT>logging.exception(e)<EOL>metadata = {}<EOL><DEDENT><DEDENT><DEDENT>return metadata<EOL>

Returns data from a package directory.
'path' should be an absolute path.


def __init__(self, for_autoencoder=False, more_data=False):

self.for_autoencoder = for_autoencoder<EOL>self._target_size = <NUM_LIT:10><EOL>logging.info("<STR_LIT>")<EOL>path = os.path.join(tempfile.gettempdir(), "<STR_LIT>")<EOL>if not os.path.exists(path):<EOL><INDENT>logging.info("<STR_LIT>")<EOL>urllib.urlretrieve (MNIST_URL, path)<EOL><DEDENT>self._train_set, self._valid_set, self._test_set = cPickle.load(gzip.open(path, '<STR_LIT:rb>'))<EOL>self._more_data = more_data<EOL>logging.info("<STR_LIT>" % len(self._train_set[<NUM_LIT:0>]))<EOL>logging.info("<STR_LIT>" % len(self._valid_set[<NUM_LIT:0>]))<EOL>logging.info("<STR_LIT>" % len(self._test_set[<NUM_LIT:0>]))<EOL>

Load MNIST dataset.
:param for_autoencoder: if this dataset is used for autoencoders


if self._train_set:<EOL><INDENT>self._train_set = pad_dataset(self._train_set, side, length)<EOL><DEDENT>if self._valid_set:<EOL><INDENT>self._valid_set = pad_dataset(self._valid_set, side, length)<EOL><DEDENT>if self._test_set:<EOL><INDENT>self._test_set = pad_dataset(self._test_set, side, length)<EOL><DEDENT>

Pad sequences to given length in the left or right side.


self._pad('<STR_LIT:left>', length)<EOL>

Pad sequences to given length in the left side.


self._pad('<STR_LIT:right>', length)<EOL>


if self._train_set:<EOL><INDENT>self._train_set = map(func, self._train_set)<EOL><DEDENT>if self._valid_set:<EOL><INDENT>self._valid_set = map(func, self._valid_set)<EOL><DEDENT>if self._test_set:<EOL><INDENT>self._test_set = map(func, self._test_set)<EOL><DEDENT>

Process all data with given function.
The scheme of function should be x,y -> x,y.


if self._train_set:<EOL><INDENT>self._train_set = self._vectorize_set(self._train_set, size)<EOL><DEDENT>if self._valid_set:<EOL><INDENT>self._valid_set = self._vectorize_set(self._valid_set, size)<EOL><DEDENT>if self._test_set:<EOL><INDENT>self._test_set = self._vectorize_set(self._test_set, size)<EOL><DEDENT>


logging.info("<STR_LIT>" % (self.__class__.__name__,<EOL>len(list(self._train_set)) if self._train_set else <NUM_LIT:0>,<EOL>len(list(self._valid_set)) if self._valid_set else <NUM_LIT:0>,<EOL>len(list(self._test_set)) if self._test_set else <NUM_LIT:0>))<EOL>


stack_len = len(bunch_stack[<NUM_LIT:0>])<EOL>for i in xrange(<NUM_LIT:0>, stack_len, self.fragment_length):<EOL><INDENT>yield np.array(map(lambda stack: stack[i: i + self.fragment_length], bunch_stack))<EOL><DEDENT>


min_len = min(map(len, bunch_stack))<EOL>for i in range(len(bunch_stack)):<EOL><INDENT>bunch_stack[i] = bunch_stack[i][:min_len]<EOL><DEDENT>


train_set = self.train_set()<EOL>if isinstance(train_set, collections.Iterable):<EOL><INDENT>return len(list(train_set))<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Return size of training data. (optional)
:rtype: number


def pad_dataset(subset, side="<STR_LIT:right>", length=-<NUM_LIT:1>):

assert length == -<NUM_LIT:1> or length > <NUM_LIT:0><EOL>if type(subset[<NUM_LIT:0>][<NUM_LIT:0>][<NUM_LIT:0>]) in [float, int, np.int64, np.int32, np.float32]:<EOL><INDENT>return _pad_2d(subset, side, length)<EOL><DEDENT>else:<EOL><INDENT>return _pad_3d(subset, side, length)<EOL><DEDENT>

Pad data set to specified length.
Parameters:
    length - max length, a just to the max length in the batch if length is -1


def stack_layer(self, layer, no_setup=False):

if layer.name:<EOL><INDENT>layer.name += "<STR_LIT>" % (len(self.layers) + <NUM_LIT:1>)<EOL><DEDENT>if not self.layers:<EOL><INDENT>layer.init(self.input_dim, no_prepare=no_setup)<EOL><DEDENT>else:<EOL><INDENT>layer.init(self.layers[-<NUM_LIT:1>].output_dim, no_prepare=no_setup)<EOL><DEDENT>self._output = layer.compute_tensor(self._output)<EOL>self._test_output = layer.compute_tensor(self._test_output)<EOL>self._hidden_outputs.append(self._output)<EOL>self.register_layer(layer)<EOL>self.layers.append(layer)<EOL>

Stack a neural layer.
:type layer: NeuralLayer
:param no_setup: whether the layer is already initialized


for layer in layers:<EOL><INDENT>self.register_layer(layer)<EOL><DEDENT>

Register multiple layers as the components of the network.
The parameter of those layers will be trained.
But the output of the layer will not be stacked.


Register the layer so that it's param will be trained.
But the output of the layer will not be stacked.


return self.layers[<NUM_LIT:0>] if self.layers else None<EOL>


for layer in layers:<EOL><INDENT>self.stack_layer(layer)<EOL><DEDENT>return self<EOL>


self.report()<EOL>

This function will be called before training.


for layer, hidden in zip(self.layers, self._hidden_outputs):<EOL><INDENT>self.training_monitors.append(('<STR_LIT>' % (layer.name), abs(hidden).mean()))<EOL><DEDENT>

Monitoring the outputs of each layer.
Useful for troubleshooting convergence problems.


params = []<EOL>params.extend(self.parameters)<EOL>params.extend(self.free_parameters)<EOL>return params<EOL>


if self.input_tensor:<EOL><INDENT>if type(self.input_tensor) == int:<EOL><INDENT>x = dim_to_var(self.input_tensor, name="<STR_LIT:x>")<EOL><DEDENT>else:<EOL><INDENT>x = self.input_tensor<EOL><DEDENT><DEDENT>else:<EOL><INDENT>x = T.matrix('<STR_LIT:x>')<EOL><DEDENT>self.input_variables.append(x)<EOL>self._output = x<EOL>self._test_output = x<EOL>


self._compile()<EOL>outs = self._compute(*x)<EOL>if self._output_keys:<EOL><INDENT>return MapDict(dict(zip(self._output_keys, outs)))<EOL><DEDENT>else:<EOL><INDENT>return outs<EOL><DEDENT>


return self._output<EOL>


return T.constant(<NUM_LIT:0>)<EOL>


def save_params(self, path, new_thread=False):

save_logger.info(path)<EOL>param_variables = self.all_parameters<EOL>params = [p.get_value().copy() for p in param_variables]<EOL>if new_thread:<EOL><INDENT>thread = Thread(target=save_network_params, args=(params, path))<EOL>thread.start()<EOL><DEDENT>else:<EOL><INDENT>save_network_params(params, path)<EOL><DEDENT>self.train_logger.save(path)<EOL>


def load_params(self, path, exclude_free_params=False):

if not os.path.exists(path): return;<EOL>logging.info("<STR_LIT>" % path)<EOL>if exclude_free_params:<EOL><INDENT>params_to_load = self.parameters<EOL><DEDENT>else:<EOL><INDENT>params_to_load = self.all_parameters<EOL><DEDENT>if path.endswith("<STR_LIT>"):<EOL><INDENT>opener = gzip.open if path.lower().endswith('<STR_LIT>') else open<EOL>handle = opener(path, '<STR_LIT:rb>')<EOL>saved_params = pickle.load(handle)<EOL>handle.close()<EOL>for target, source in zip(params_to_load, saved_params):<EOL><INDENT>logging.info('<STR_LIT>', target.name, source.shape)<EOL>target.set_value(source)<EOL><DEDENT><DEDENT>elif path.endswith("<STR_LIT>"):<EOL><INDENT>arrs = np.load(path)<EOL>for target, idx in zip(params_to_load, range(len(arrs.keys()))):<EOL><INDENT>source = arrs['<STR_LIT>' % idx]<EOL>logging.info('<STR_LIT>', target.name, source.shape)<EOL>target.set_value(source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % path)<EOL><DEDENT>self.train_logger.load(path)<EOL>


logging.info("<STR_LIT>", "<STR_LIT:U+0020>".join(map(str, self.input_variables)))<EOL>logging.info("<STR_LIT>", "<STR_LIT:U+0020>".join(map(str, self.target_variables)))<EOL>logging.info("<STR_LIT>", "<STR_LIT:U+0020>".join(map(str, self.all_parameters)))<EOL>logging.info("<STR_LIT>", self.parameter_count)<EOL>


for cb in self.epoch_callbacks:<EOL><INDENT>cb()<EOL><DEDENT>


for cb in self.training_callbacks:<EOL><INDENT>cb()<EOL><DEDENT>


def __init__(self, input_dim, rep_dim, input_tensor=None):

super(AutoEncoder, self).__init__(input_dim, input_tensor=input_tensor)<EOL>self.rep_dim = rep_dim<EOL>self.encoding_layes = []<EOL>self.decoding_layers = []<EOL>self.encoding_network = None<EOL>self.decoding_network = None<EOL>

Create an auto-encoder.
:param input_dim: dimension of the input layer
:param rep_dim: dimension of the representation layer
:param input_tensor:
:return:


self.stack(*layers)<EOL>self.encoding_layes.extend(layers)<EOL>

Stack encoding layers, this must be done before stacking decoding layers.


self.stack(*layers)<EOL>self.decoding_layers.extend(layers)<EOL>


if not self.encoding_network:<EOL><INDENT>self.encoding_network = NeuralNetwork(self.input_dim, self.input_tensor)<EOL>self.encoding_network.input_variables = self.input_variables<EOL>for layer in self.encoding_layes:<EOL><INDENT>self.encoding_network.stack_layer(layer, no_setup=True)<EOL><DEDENT><DEDENT>return self.encoding_network.compute(*x)<EOL>


if not self.rep_dim:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if not self.decoding_network:<EOL><INDENT>self.decoding_network = NeuralNetwork(self.rep_dim)<EOL>for layer in self.decoding_layers:<EOL><INDENT>self.decoding_network.stack_layer(layer, no_setup=True)<EOL><DEDENT><DEDENT>return self.decoding_network.compute(x)<EOL>


<EOL>flattened = [T.as_tensor_variable(t).flatten() for t in tensors]<EOL>flattened = [(t if t.ndim > <NUM_LIT:0> else t.dimshuffle('<STR_LIT:x>'))<EOL>for t in flattened]<EOL>joined = T.join(<NUM_LIT:0>, *flattened)<EOL>return T.sqrt(T.sqr(joined).sum())<EOL>

Get the L2 norm of multiple tensors.
This function is taken from blocks.


def __init__(self, network, config=None, batch_size=<NUM_LIT:20>):

super(DelayedBatchSGDTrainer, self).__init__(network, config)<EOL>self.learning_rate = self.config.learning_rate<EOL>self.batch_size = batch_size<EOL>logging.info('<STR_LIT>', self.__class__.__name__)<EOL>network_updates = list(network.updates) + list(network._learning_updates)<EOL>learning_updates = list(self.learning_updates())<EOL>update_list = network_updates + learning_updates<EOL>logging.info("<STR_LIT>" % "<STR_LIT:U+0020>".join(map(str, [x[<NUM_LIT:0>] for x in network_updates])))<EOL>logging.info("<STR_LIT>" % "<STR_LIT:U+0020>".join(map(str, [x[<NUM_LIT:0>] for x in learning_updates])))<EOL>self.learning_func = theano.function(<EOL>network.inputs,<EOL>self.training_variables,<EOL>updates=update_list, allow_input_downcast=True, mode=theano.Mode(linker=THEANO_LINKER))<EOL>

Create a SGD trainer.
:type network:
:type config: deepy.conf.TrainerConfig
:return:


def optimize_updates(params, gradients, config=None, shapes=None):

if config and isinstance(config, dict):<EOL><INDENT>config = TrainerConfig(config)<EOL><DEDENT>if config:<EOL><INDENT>clip_value = config.get("<STR_LIT>", None)<EOL>if clip_value:<EOL><INDENT>clip_constant = T.constant(clip_value, dtype=FLOATX)<EOL>if config.avoid_compute_embed_norm:<EOL><INDENT>grad_norm = multiple_l2_norm([t[<NUM_LIT:1>] for t in zip(params, gradients) if not t[<NUM_LIT:0>].name.startswith("<STR_LIT>")])<EOL><DEDENT>else:<EOL><INDENT>grad_norm = multiple_l2_norm(gradients)<EOL><DEDENT>isnan = T.or_(T.isnan(grad_norm), T.isinf(grad_norm))<EOL>multiplier = ifelse(grad_norm < clip_constant,<EOL>T.constant(<NUM_LIT:1.>, dtype=FLOATX), clip_constant / (grad_norm + EPSILON))<EOL>clipped_gradients = []<EOL>for param, g in zip(params, gradients):<EOL><INDENT>g = multiplier * g<EOL>if config.avoid_nan:<EOL><INDENT>g = T.switch(isnan, np.float32(<NUM_LIT:0.1>) * param, g)<EOL><DEDENT>if config.gradient_tolerance:<EOL><INDENT>g = ifelse(grad_norm > config.gradient_tolerance, T.zeros_like(g) + EPSILON, g)<EOL><DEDENT>clipped_gradients.append(g)<EOL><DEDENT>gradients = clipped_gradients<EOL><DEDENT><DEDENT>if config and config.weight_l2:<EOL><INDENT>regularized_gradients = []<EOL>for param, grad in zip(params, gradients):<EOL><INDENT>grad = grad + (<NUM_LIT:2> * config.weight_l2 * param)<EOL>regularized_gradients.append(grad)<EOL><DEDENT>gradients = regularized_gradients<EOL><DEDENT>if config and config.avoid_nan and not config.gradient_clipping:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>new_gradients = []<EOL>for grad in gradients:<EOL><INDENT>new_grad = ifelse(T.isnan(grad).any(), T.zeros_like(grad) + EPSILON, grad)<EOL>new_gradients.append(new_grad)<EOL><DEDENT>gradients = new_gradients<EOL><DEDENT>method = "<STR_LIT>"<EOL>if config:<EOL><INDENT>method = config.get("<STR_LIT>", method).upper()<EOL><DEDENT>func = None<EOL>if method in ["<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"]:<EOL><INDENT>from cores.ada_family import ada_family_core<EOL>func = ada_family_core<EOL><DEDENT>elif method == "<STR_LIT>":<EOL><INDENT>from cores.adam import adam_core<EOL>func = adam_core<EOL><DEDENT>elif method == "<STR_LIT>":<EOL><INDENT>from cores.rmsprop import rmsprop_core<EOL>func = rmsprop_core<EOL><DEDENT>elif method == "<STR_LIT>":<EOL><INDENT>from cores.momentum import momentum_core<EOL>func = momentum_core<EOL><DEDENT>if not func:<EOL><INDENT>raise NotImplementedError("<STR_LIT>" % method)<EOL><DEDENT>logging.info("<STR_LIT>" % (method, str(params)))<EOL>free_parameters = []<EOL>return_vals = wrap_core(func, config, params, gradients)<EOL>if type(return_vals) == list and type(return_vals[<NUM_LIT:0>]) == list:<EOL><INDENT>updates, free_parameters = return_vals<EOL><DEDENT>else:<EOL><INDENT>updates = return_vals<EOL><DEDENT>if config and not config.record_free_params:<EOL><INDENT>free_parameters = []<EOL><DEDENT>if config.weight_bound:<EOL><INDENT>logging.info("<STR_LIT>" % config.weight_bound)<EOL>new_updates = []<EOL>for param, update_value in updates:<EOL><INDENT>bounded_value = (update_value * (T.abs_(update_value) <= config.weight_bound) +<EOL>config.weight_bound * (update_value > config.weight_bound) +<EOL>-config.weight_bound * (update_value < -config.weight_bound))<EOL>new_updates.append((param, bounded_value))<EOL><DEDENT>updates = new_updates<EOL><DEDENT>return updates, free_parameters<EOL>

General optimization function for Theano.
Parameters:
    params - parameters
    gradients - gradients
    config - training config
Returns:
    Theano updates
:type config: deepy.TrainerConfig or dict


gs = [dim_to_var(p.ndim) for p in params]<EOL>updates, _ = optimize_updates(params, gs, config)<EOL>return theano.function(gs, [], updates=updates)<EOL>

Create a optimizing function receives gradients.
Parameters:
    params - parameters
    config - training configuration
Returns:
    updating function receives gradients


def __init__(self, patience=<NUM_LIT:3>, anneal_times=<NUM_LIT:4>):

self._iter = <NUM_LIT:0><EOL>self._annealed_iter = <NUM_LIT:0><EOL>self._patience = patience<EOL>self._anneal_times = anneal_times<EOL>self._annealed_times = <NUM_LIT:0><EOL>self._learning_rate = <NUM_LIT:0><EOL>if type(self._learning_rate) == float:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>


self._iter += <NUM_LIT:1><EOL>if self._iter - max(self._trainer.best_iter, self._annealed_iter) >= self._patience:<EOL><INDENT>if self._annealed_times >= self._anneal_times:<EOL><INDENT>logging.info("<STR_LIT>")<EOL>self._trainer.exit()<EOL><DEDENT>else:<EOL><INDENT>self._trainer.set_params(*self._trainer.best_params)<EOL>self._learning_rate.set_value(self._learning_rate.get_value() * <NUM_LIT:0.5>)<EOL>self._annealed_times += <NUM_LIT:1><EOL>self._annealed_iter = self._iter<EOL>logging.info("<STR_LIT>" % self._learning_rate.get_value())<EOL><DEDENT><DEDENT>


return graph.shared(value, name="<STR_LIT>")<EOL>


self._iter = <NUM_LIT:0><EOL>self._patience = end_at<EOL>


self._iter += <NUM_LIT:1><EOL>logging.info("<STR_LIT>".format(self._patience - self._iter))<EOL>if self._iter >= self._patience:<EOL><INDENT>self._trainer.exit()<EOL><DEDENT>


def ada_family_core(params, gparams, learning_rate = <NUM_LIT>, eps= <NUM_LIT>, rho=<NUM_LIT>, method="<STR_LIT>",<EOL>beta=<NUM_LIT:0.0>, gsum_regularization = <NUM_LIT>):

_, _, _, args = inspect.getargvalues(inspect.currentframe())<EOL>logging.info("<STR_LIT>" % str(args.items()))<EOL>free_parameters = []<EOL>if method == "<STR_LIT>":<EOL><INDENT>method = "<STR_LIT>"<EOL>gsum_regularization = <NUM_LIT:0><EOL><DEDENT>oneMinusBeta = <NUM_LIT:1> - beta<EOL>gsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="<STR_LIT>" % param.name) if (method == '<STR_LIT>' or method == '<STR_LIT>') else None for param in params]<EOL>xsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="<STR_LIT>" % param.name) if method == '<STR_LIT>' else None for param in params]<EOL>if method == '<STR_LIT>':<EOL><INDENT>for gsum in gsums:<EOL><INDENT>gsum.set_value(gsum.get_value() ** <NUM_LIT:0>)<EOL><DEDENT><DEDENT>updates = OrderedDict()<EOL>for gparam, param, gsum, xsum in zip(gparams, params, gsums, xsums):<EOL><INDENT>if method == '<STR_LIT>':<EOL><INDENT>updates[gsum] = rho * gsum + (<NUM_LIT:1.> - rho) * (gparam **<NUM_LIT:2>)<EOL>dparam = -T.sqrt((xsum + eps) / (updates[gsum] + eps)) * gparam<EOL>updates[xsum] =rho * xsum + (<NUM_LIT:1.> - rho) * (dparam **<NUM_LIT:2>)<EOL>updates[param] = param * oneMinusBeta + dparam<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>updates[gsum] = gsum + (gparam **<NUM_LIT:2>) - gsum_regularization * gsum<EOL>updates[param] =  param * oneMinusBeta - learning_rate * (gparam / (T.sqrt(updates[gsum] + eps)))<EOL><DEDENT>else:<EOL><INDENT>updates[param] = param * oneMinusBeta - gparam * learning_rate<EOL><DEDENT><DEDENT>if method == '<STR_LIT>':<EOL><INDENT>free_parameters.extend(gsums + xsums)<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>free_parameters.extend(gsums)<EOL><DEDENT>for k in updates:<EOL><INDENT>if updates[k].dtype != FLOATX:<EOL><INDENT>updates[k] = updates[k].astype(FLOATX)<EOL><DEDENT><DEDENT>return updates.items(), free_parameters<EOL>


def nag_core(params, J, momentum=<NUM_LIT>, learning_rate=<NUM_LIT>):

<EOL>for param in params:<EOL><INDENT>step = theano.shared(np.zeros_like(param.get_value()), name=param.name + '<STR_LIT>')<EOL>velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '<STR_LIT>')<EOL>yield step, momentum * velocity<EOL>yield param, param + step<EOL>yield velocity, step - learning_rate * T.grad(J, param)<EOL>yield param, param + velocity - step<EOL><DEDENT>

Nesterov's Accelerated Gradient (NAG).
See http://www.cs.toronto.edu/~fritz/absps/momentum.pdf .
Still unfinished


def rprop_core(params, gradients, rprop_increase=<NUM_LIT>, rprop_decrease=<NUM_LIT>, rprop_min_step=<NUM_LIT:0>, rprop_max_step=<NUM_LIT:100>,<EOL>learning_rate=<NUM_LIT>):

for param, grad in zip(params, gradients):<EOL><INDENT>grad_tm1 = theano.shared(np.zeros_like(param.get_value()), name=param.name + '<STR_LIT>')<EOL>step_tm1 = theano.shared(np.zeros_like(param.get_value()) + learning_rate, name=param.name+ '<STR_LIT>')<EOL>test = grad * grad_tm1<EOL>same = T.gt(test, <NUM_LIT:0>)<EOL>diff = T.lt(test, <NUM_LIT:0>)<EOL>step = T.minimum(rprop_max_step, T.maximum(rprop_min_step, step_tm1 * (<EOL>T.eq(test, <NUM_LIT:0>) +<EOL>same * rprop_increase +<EOL>diff * rprop_decrease)))<EOL>grad = grad - diff * grad<EOL>yield param, param - T.sgn(grad) * step<EOL>yield grad_tm1, grad<EOL>yield step_tm1, step<EOL><DEDENT>

Rprop optimizer.
See http://sci2s.ugr.es/keel/pdf/algorithm/articulo/2003-Neuro-Igel-IRprop+.pdf.


def rmsprop_core(params, gradients, momentum=<NUM_LIT>, learning_rate=<NUM_LIT>):

for param, grad in zip(params, gradients):<EOL><INDENT>rms_ = theano.shared(np.zeros_like(param.get_value()), name=param.name + '<STR_LIT>')<EOL>rms = momentum * rms_ + (<NUM_LIT:1> - momentum) * grad * grad<EOL>yield rms_, rms<EOL>yield param, param - learning_rate * grad / T.sqrt(rms + <NUM_LIT>)<EOL><DEDENT>


def momentum_core(params, gradients, momentum=<NUM_LIT>, learning_rate=<NUM_LIT>):

free_parameters = []<EOL>updates = []<EOL>for param, grad in zip(params, gradients):<EOL><INDENT>delta = learning_rate * grad<EOL>velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '<STR_LIT>')<EOL>updates.append((velocity, momentum * velocity - delta))<EOL>updates.append((param, param + velocity))<EOL>free_parameters.append(velocity)<EOL><DEDENT>return updates, free_parameters<EOL>


def __init__(self, network, config=None, validator=None, annealer=None):

super(NeuralTrainer, self).__init__()<EOL>self.config = None<EOL>if isinstance(config, TrainerConfig):<EOL><INDENT>self.config = config<EOL><DEDENT>elif isinstance(config, dict):<EOL><INDENT>self.config = TrainerConfig(config)<EOL><DEDENT>else:<EOL><INDENT>self.config = TrainerConfig()<EOL><DEDENT>if type(self.config.learning_rate) == float:<EOL><INDENT>self.config.learning_rate = np.array(self.config.learning_rate, dtype=env.FLOATX)<EOL><DEDENT>self.model = self.network = network<EOL>self.network.prepare_training()<EOL>self._setup_costs()<EOL>self.evaluation_func = None<EOL>self.validation_frequency = self.config.validation_frequency<EOL>self.min_improvement = self.config.min_improvement<EOL>self.patience = self.config.patience<EOL>self._iter_controllers = []<EOL>self._epoch_controllers = []<EOL>if annealer:<EOL><INDENT>annealer.bind(self)<EOL>self._epoch_controllers.append(annealer)<EOL><DEDENT>if validator:<EOL><INDENT>validator.bind(self)<EOL>self._iter_controllers.append(validator)<EOL><DEDENT>self.best_cost = <NUM_LIT><EOL>self.best_epoch = <NUM_LIT:0><EOL>self.best_params = self.copy_params()<EOL>self._skip_batches = <NUM_LIT:0><EOL>self._skip_epochs = <NUM_LIT:0><EOL>self._progress = <NUM_LIT:0><EOL>self.last_cost = <NUM_LIT:0><EOL>self.last_run_costs = None<EOL>self._report_time = True<EOL>self._epoch = <NUM_LIT:0><EOL>self._current_train_set = None<EOL>self._current_valid_set = None<EOL>self._current_test_set = None<EOL>self._ended = False<EOL>

Basic neural network trainer.
:type network: deepy.NeuralNetwork
:type config: deepy.conf.TrainerConfig
:return:


def skip(self, n_batches, n_epochs=<NUM_LIT:0>):

logging.info("<STR_LIT>" % (n_epochs, n_batches))<EOL>self._skip_batches = n_batches<EOL>self._skip_epochs = n_epochs<EOL>


return self._epoch<EOL>


self.network.load_params(path, exclude_free_params=exclude_free_params)<EOL>self.best_params = self.copy_params()<EOL>if self.network.train_logger.progress() > <NUM_LIT:0> or self.network.train_logger.epoch() > <NUM_LIT:0>:<EOL><INDENT>self.skip(self.network.train_logger.progress(), self.network.train_logger.epoch() - <NUM_LIT:1>)<EOL><DEDENT>

Load parameters for the training.
This method can load free parameters and resume the training progress.


self._ended = True<EOL>


for controller in controllers:<EOL><INDENT>if isinstance(controller, TrainingController):<EOL><INDENT>controller.bind(self)<EOL><DEDENT>self._iter_controllers.append(controller)<EOL><DEDENT>

Add iteration callbacks function (receives an argument of the trainer).
:param controllers: can be a `TrainingController` or a function.
:type funcs: list of TrainingContoller


for controller in controllers:<EOL><INDENT>if isinstance(controller, TrainingController):<EOL><INDENT>controller.bind(self)<EOL><DEDENT>self._epoch_controllers.append(controller)<EOL><DEDENT>

Add epoch callbacks function.
:param controllers: can be a `TrainingController` or a function.


Update the parameters and return the cost with given data points.
:param variables:
:return:


def _run_train(self, epoch, train_set, train_size=None):

self.network.train_logger.record_epoch(epoch + <NUM_LIT:1>)<EOL>costs = self.train_step(train_set, train_size)<EOL>if not epoch % self.config.monitor_frequency:<EOL><INDENT>self.report(dict(costs), "<STR_LIT:train>", epoch)<EOL><DEDENT>self.last_run_costs = costs<EOL>return costs<EOL>


def _run_valid(self, epoch, valid_set, dry_run=False, save_path=None):

costs = self.valid_step(valid_set)<EOL>_, J = costs[<NUM_LIT:0>]<EOL>new_best = False<EOL>if self.best_cost - J > self.best_cost * self.min_improvement:<EOL><INDENT>self.best_params = self.copy_params()<EOL>new_best = True<EOL>if not dry_run:<EOL><INDENT>self.best_cost = J<EOL>self.best_epoch = epoch<EOL><DEDENT>self.save_checkpoint(save_path)<EOL><DEDENT>self.report(dict(costs), type="<STR_LIT>", epoch=<NUM_LIT:0> if dry_run else epoch, new_best=new_best)<EOL>self.last_run_costs = costs<EOL>return epoch - self.best_epoch < self.patience<EOL>

Run one valid iteration, return true if to continue training.


def report(self, score_map, type="<STR_LIT>", epoch=-<NUM_LIT:1>, new_best=False):

type_str = type<EOL>if len(type_str) < <NUM_LIT:5>:<EOL><INDENT>type_str += "<STR_LIT:U+0020>" * (<NUM_LIT:5> - len(type_str))<EOL><DEDENT>info = "<STR_LIT:U+0020>".join("<STR_LIT>" % el for el in score_map.items())<EOL>current_epoch = epoch if epoch > <NUM_LIT:0> else self.current_epoch()<EOL>epoch_str = "<STR_LIT>".format(current_epoch + <NUM_LIT:1>)<EOL>if epoch < <NUM_LIT:0>:<EOL><INDENT>epoch_str = "<STR_LIT>"<EOL>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL><DEDENT>marker = "<STR_LIT>" if new_best else "<STR_LIT>"<EOL>message = "<STR_LIT>".format(type_str, epoch_str, info, marker)<EOL>self.network.train_logger.record(message)<EOL>logging.info(message)<EOL>

Report the scores and record them in the log.


if data_split == '<STR_LIT:train>':<EOL><INDENT>return self._current_train_set<EOL><DEDENT>elif data_split == '<STR_LIT>':<EOL><INDENT>return self._current_valid_set<EOL><DEDENT>elif data_split == '<STR_LIT:test>':<EOL><INDENT>return self._current_test_set<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>


self._progress = progress<EOL>

Record current progress in the training[1;3B].


return self._progress<EOL>


def __init__(self, network, config=None):

super(CustomizeTrainer, self).__init__(network, config)<EOL>


params = self.training_params()<EOL>gradients = self.get_gradients(params)<EOL>return self.optimization_updates(params, gradients)<EOL>


params = self.network.parameters<EOL>if self.config.fixed_parameters:<EOL><INDENT>logging.info("<STR_LIT>" % "<STR_LIT:U+002CU+0020>".join(map(str, self.config.fixed_parameters)))<EOL>params = [p for p in params if p not in self.config.fixed_parameters]<EOL><DEDENT>return params<EOL>


return T.grad(self.cost, params)<EOL>


def optimization_updates(self, params, gradients):

updates, free_parameters = optimize_updates(params, gradients, self.config)<EOL>self.network.free_parameters.extend(free_parameters)<EOL>logging.info("<STR_LIT>" % len(free_parameters))<EOL>return updates<EOL>


network_updates = list(self.network.updates) + list(self.network.training_updates)<EOL>learning_updates = list(self._learning_updates())<EOL>update_list = network_updates + learning_updates<EOL>logging.info("<STR_LIT>" % "<STR_LIT:U+0020>".join(map(str, [x[<NUM_LIT:0>] for x in network_updates])))<EOL>logging.info("<STR_LIT>" % "<STR_LIT:U+0020>".join(map(str, [x[<NUM_LIT:0>] for x in learning_updates])))<EOL>variables = self.network.input_variables + self.network.target_variables<EOL>givens = None<EOL>return theano.function(<EOL>variables,<EOL>map(lambda v: theano.Out(v, borrow=True), self.training_variables),<EOL>updates=update_list, allow_input_downcast=True,<EOL>mode=self.config.get("<STR_LIT>", None),<EOL>givens=givens)<EOL>

Get the learning function.
:param func:
:return:


self._trainer = trainer<EOL>


return False<EOL>


def __init__(self, valid_model=None, data_split='<STR_LIT>', freq=<NUM_LIT>, save_path=None, criteria='<STR_LIT>',<EOL>smaller_is_better=True):

self._model = valid_model<EOL>self._data_split = data_split<EOL>self._freq = freq<EOL>self._save_path = save_path<EOL>self._criteria = criteria<EOL>self._smaller_is_better = smaller_is_better<EOL>self._best_criteria = None<EOL>self._counter = <NUM_LIT:0><EOL>


cri_val = cost_map[self._criteria]<EOL>if self._best_criteria is None:<EOL><INDENT>self._best_criteria = cri_val<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>if self._smaller_is_better and cri_val < self._best_criteria:<EOL><INDENT>self._best_criteria = cri_val<EOL>return True<EOL><DEDENT>elif not self._smaller_is_better and cri_val > self._best_criteria:<EOL><INDENT>self._best_criteria = cri_val<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>

Compare to previous records and return whether the given cost is a new best.
:return: True if the given cost is a new best


return self._model.compute(*x)<EOL>

Compute with the validation model given data x.


output_vars = self.compute(*data_x)<EOL>return self._extract_costs(output_vars)<EOL>

Run the model with validation data and return costs.


self._counter += <NUM_LIT:1><EOL>if self._counter % self._freq == <NUM_LIT:0>:<EOL><INDENT>cnt = <NUM_LIT:0.><EOL>sum_map = defaultdict(float)<EOL>for x in self._trainer.get_data(self._data_split):<EOL><INDENT>val_map = self.run(x)<EOL>if not isinstance(val_map, dict):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>for k, val in val_map.items():<EOL><INDENT>sum_map[k] += val<EOL><DEDENT>cnt += <NUM_LIT:1><EOL><DEDENT>for k in sum_map:<EOL><INDENT>sum_map[k] /= cnt<EOL><DEDENT>new_best = self.compare(sum_map)<EOL>self._trainer.report(sum_map, self._data_split, new_best=new_best)<EOL>if new_best:<EOL><INDENT>self._trainer.save_checkpoint(self._save_path)<EOL><DEDENT><DEDENT>

This function will be called after each iteration.


return T.dot(inputs, self.Ua)<EOL>

Precompute partial values in the score function.


def compute_alignments(self, prev_state, precomputed_values, mask=None):

WaSp = T.dot(prev_state, self.Wa)<EOL>UaH = precomputed_values<EOL>if UaH.ndim == <NUM_LIT:2>:<EOL><INDENT>preact = WaSp[:, None, :] + UaH[None, :, :]<EOL><DEDENT>else:<EOL><INDENT>preact = WaSp[:, None, :] + UaH<EOL><DEDENT>act = T.activate(preact, '<STR_LIT>')<EOL>align_scores = T.dot(act, self.Va)  <EOL>if mask:<EOL><INDENT>mask = (<NUM_LIT:1> - mask) * -<NUM_LIT><EOL>if align_scores.ndim == <NUM_LIT:3>:<EOL><INDENT>align_scores += mask[None, :]<EOL><DEDENT>else:<EOL><INDENT>align_scores += mask<EOL><DEDENT><DEDENT>align_weights = T.nnet.softmax(align_scores)<EOL>return align_weights<EOL>

Compute the alignment weights based on the previous state.


def compute_context_vector(self, prev_state, inputs, precomputed_values=None, mask=None):

precomputed_values = precomputed_values if precomputed_values else self.precompute(inputs)<EOL>align_weights = self.compute_alignments(prev_state, precomputed_values, mask)<EOL>context_vector = T.sum(align_weights[:, :, None] * inputs, axis=<NUM_LIT:1>)<EOL>return context_vector<EOL>

Compute the context vector with soft attention.


super(Concatenate, self).__init__("<STR_LIT>")<EOL>self.axis = axis<EOL>

:type layer1: NeuralLayer
:type layer2: NeuralLayer


def __init__(self, func, dim=<NUM_LIT:0>):

super(Combine, self).__init__("<STR_LIT>")<EOL>self.func = func<EOL>if dim > <NUM_LIT:0>:<EOL><INDENT>self.output_dim = dim<EOL><DEDENT>


self.name = name if name else self.__class__.__name__<EOL>self.input_dim = <NUM_LIT:0><EOL>self.input_dims = [<NUM_LIT:0>]<EOL>self.output_dim = <NUM_LIT:0><EOL>self.output_dims= [<NUM_LIT:0>]<EOL>self._linked_block = None<EOL>self.initialized = False<EOL>self.updates = []<EOL>self.training_updates = []<EOL>self.free_parameters = []<EOL>self.parameters = []<EOL>self.training_monitors = []<EOL>self.testing_monitors = []<EOL>self._registered_monitors = set()<EOL>self._registered_updates = set()<EOL>self._registered_training_updates = set()<EOL>self.external_inputs = []<EOL>self.external_targets = []<EOL>self.parameter_count = <NUM_LIT:0><EOL>self.epoch_callbacks = []<EOL>self.training_callbacks = []<EOL>self.testing_callbacks = []<EOL>


def init(self, input_dim=<NUM_LIT:0>, input_dims=None, no_prepare=False):

if self.initialized:<EOL><INDENT>return<EOL><DEDENT>if input_dims:<EOL><INDENT>self.input_dims = input_dims<EOL>self.input_dim = input_dims[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>self.input_dim = input_dim<EOL>self.input_dims = [input_dims]<EOL><DEDENT>if self.output_dim == <NUM_LIT:0>:<EOL><INDENT>self.output_dim = self.input_dim<EOL><DEDENT>self.initialized = True<EOL>if not no_prepare:<EOL><INDENT>self.prepare()<EOL><DEDENT>return self<EOL>

Initialize the layer.
:param no_prepare: avoid calling preparation function


from deepy.core.neural_var import NeuralVariable<EOL>from deepy.core.graph import graph<EOL>if type(inputs[<NUM_LIT:0>]) != NeuralVariable:<EOL><INDENT>raise SystemError("<STR_LIT>")<EOL><DEDENT>dims = [t.dim() for t in inputs]<EOL>if len(inputs) == <NUM_LIT:1>:<EOL><INDENT>self.init(input_dim=dims[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>self.init(input_dims=dims)<EOL><DEDENT>if self.parameters and not self._linked_block:<EOL><INDENT>self.belongs_to(graph.default_block())<EOL><DEDENT>train_kwargs, _, _ = convert_to_theano_var(kwargs)<EOL>output = self.compute_tensor(*[t.tensor for t in inputs], **train_kwargs)<EOL>if type(output) != list and type(output) != tuple:<EOL><INDENT>return NeuralVariable(output, dim=self.output_dim)<EOL><DEDENT>else:<EOL><INDENT>return [NeuralVariable(*item) for item in zip(output, self.output_dims)]<EOL><DEDENT>

Compute based on NeuralVariable.
:type inputs:  list of NeuralVariable
:return: NeuralVariable


Prepare function will be called after connected.


@neural_computation_prefer_tensor<EOL><INDENT>def compute_tensor(self, *args, **kwargs):<DEDENT>

raise NotImplementedError("<STR_LIT>" % self.name)<EOL>


if self._linked_block:<EOL><INDENT>raise SystemError("<STR_LIT>".format(self.name, self._linked_block.name))<EOL><DEDENT>self._linked_block = block<EOL>block.register_layer(self)<EOL>return self<EOL>

Let the given block or network manage the parameters of this layer.
:param block: Block or NeuralNetwork
:return: NeuralLayer


self.register_inner_layers(*layers)<EOL>


for param in parameters:<EOL><INDENT>self.parameter_count += np.prod(param.get_value().shape)<EOL><DEDENT>self.parameters.extend(parameters)<EOL>


return self.free_parameters.extend(free_parameters)<EOL>

Register free parameters, which means their value will not be learned by trainer.


for key, node in updates:<EOL><INDENT>if key not in self._registered_updates:<EOL><INDENT>self.updates.append((key, node))<EOL>self._registered_updates.add(key)<EOL><DEDENT><DEDENT>

Register updates that will be executed in each iteration.


for key, node in updates:<EOL><INDENT>if key not in self._registered_training_updates:<EOL><INDENT>self.training_updates.append((key, node))<EOL>self._registered_training_updates.add(key)<EOL><DEDENT><DEDENT>

Register updates that will only be executed in training phase.


for key, node in monitors:<EOL><INDENT>if key not in self._registered_monitors:<EOL><INDENT>node *= <NUM_LIT:1.0> <EOL>self.training_monitors.append((key, node))<EOL>self.testing_monitors.append((key, node))<EOL>self._registered_monitors.add(key)<EOL><DEDENT><DEDENT>

Register monitors they should be tuple of name and Theano variable.


self.external_inputs.extend(variables)<EOL>


self.external_targets.extend(variables)<EOL>


self.training_callbacks.extend(callbacks)<EOL>

Register callback for each iteration in the training.


self.epoch_callbacks.extend(callbacks)<EOL>

Register callback which will be called after epoch finished.


self.name = name<EOL>

Set the name of this layer.
This will be the key of saved parameters.


name = kwargs['<STR_LIT:name>'] if '<STR_LIT:name>' in kwargs else "<STR_LIT>".format(self._BLOCK_COUNT + <NUM_LIT:1>)<EOL>super(Block, self).__init__(name)<EOL>self._BLOCK_COUNT += <NUM_LIT:1><EOL>self.layers = list(layers)<EOL>self.fixed = False<EOL>

Create a new parameter block with some layers.
You can also specify a name through kwargs.


if not self.fixed:<EOL><INDENT>for layer in self.layers:<EOL><INDENT>if not layer.initialized:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self.register_inner_layers(layer)<EOL><DEDENT>self.fixed = True<EOL><DEDENT>

Fix the block, register all the parameters of sub layers.
:return:


for layer in layers:<EOL><INDENT>self.register_layer(layer)<EOL><DEDENT>

Register many connected layers.
:type layers: list of NeuralLayer


if self.fixed:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self.layers.append(layer)<EOL>

Register one connected layer.
:type layer: NeuralLayer


def __init__(self, output_dim=None, num_pieces=<NUM_LIT:4>, init=None, linear_transform=True):

super(Maxout, self).__init__("<STR_LIT>")<EOL>self.num_pieces = num_pieces<EOL>self.output_dim = output_dim<EOL>self.linear_transform = linear_transform<EOL>self.init = init<EOL>


@abstractmethod<EOL><INDENT>def merge_inputs(self, input_var, additional_inputs=None):<DEDENT>

Merge inputs and return a map, which will be passed to core_step.
:type input_var: T.var
:param additional_inputs: list
:rtype: dict


@neural_computation<EOL><INDENT>def compute_step(self, state, lstm_cell=None, input=None, additional_inputs=None):<DEDENT>

if not self.initialized:<EOL><INDENT>input_dim = None<EOL>if input and hasattr(input.tag, '<STR_LIT>'):<EOL><INDENT>input_dim = input.tag.last_dim<EOL><DEDENT>self.init(input_dim)<EOL><DEDENT>input_map = self.merge_inputs(input, additional_inputs=additional_inputs)<EOL>input_map.update({"<STR_LIT:state>": state, "<STR_LIT>": lstm_cell})<EOL>output_map = self.compute_new_state(input_map)<EOL>outputs = [output_map.pop("<STR_LIT:state>")]<EOL>outputs += output_map.values()<EOL>for tensor in outputs:<EOL><INDENT>tensor.tag.last_dim = self.hidden_size<EOL><DEDENT>if len(outputs) == <NUM_LIT:1>:<EOL><INDENT>return outputs[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return outputs<EOL><DEDENT>

Compute one step in the RNN.
:return: one variable for RNN and GRU, multiple variables for LSTM


@neural_computation<EOL><INDENT>def get_initial_states(self, input_var, init_state=None):<DEDENT>

initial_states = {}<EOL>for state in self.state_names:<EOL><INDENT>if state != "<STR_LIT:state>" or not init_state:<EOL><INDENT>if self._input_type == '<STR_LIT>' and input_var.ndim == <NUM_LIT:2>:<EOL><INDENT>init_state = T.alloc(np.cast[env.FLOATX](<NUM_LIT:0.>), self.hidden_size)<EOL><DEDENT>else:<EOL><INDENT>init_state = T.alloc(np.cast[env.FLOATX](<NUM_LIT:0.>), input_var.shape[<NUM_LIT:0>], self.hidden_size)<EOL><DEDENT><DEDENT>initial_states[state] = init_state<EOL><DEDENT>return initial_states<EOL>


@neural_computation<EOL><INDENT>def get_step_inputs(self, input_var, states=None, mask=None, additional_inputs=None):<DEDENT>

step_inputs = {}<EOL>if self._input_type == "<STR_LIT>":<EOL><INDENT>if not additional_inputs:<EOL><INDENT>additional_inputs = []<EOL><DEDENT>if mask:<EOL><INDENT>step_inputs['<STR_LIT>'] = mask.dimshuffle(<NUM_LIT:1>, <NUM_LIT:0>)<EOL><DEDENT>step_inputs.update(self.merge_inputs(input_var, additional_inputs=additional_inputs))<EOL><DEDENT>else:<EOL><INDENT>if additional_inputs:<EOL><INDENT>step_inputs.update(self.merge_inputs(None, additional_inputs=additional_inputs))<EOL><DEDENT><DEDENT>if states:<EOL><INDENT>for name in self.state_names:<EOL><INDENT>step_inputs[name] = states[name]<EOL><DEDENT><DEDENT>return step_inputs<EOL>


from deepy.core.neural_var import NeuralVariable<EOL>if type(obj) == tuple:<EOL><INDENT>return tuple(convert_to_theano_var(list(obj)))<EOL><DEDENT>if type(obj) == list:<EOL><INDENT>unpacked_list = map(convert_to_theano_var, obj)<EOL>normal_list = []<EOL>test_list = []<EOL>theano_var_found = False<EOL>neural_var_found = False<EOL>for normal_var, tensor_found, neural_found in unpacked_list:<EOL><INDENT>normal_list.append(normal_var)<EOL>if tensor_found: theano_var_found = True<EOL>if neural_found: neural_var_found = True<EOL><DEDENT>return normal_list, theano_var_found, neural_var_found<EOL><DEDENT>elif type(obj) == dict:<EOL><INDENT>normal_map = {}<EOL>theano_var_found = False<EOL>neural_var_found = False<EOL>for key in obj:<EOL><INDENT>normal_var, tensor_found, neural_found = convert_to_theano_var(obj[key])<EOL>normal_map[key] = normal_var<EOL>if tensor_found: theano_var_found = True<EOL>if neural_found: neural_var_found = True<EOL><DEDENT>return normal_map, theano_var_found, neural_var_found<EOL><DEDENT>elif type(obj) == MapDict:<EOL><INDENT>normal_map = {}<EOL>theano_var_found = False<EOL>neural_var_found = False<EOL>for key in obj:<EOL><INDENT>normal_var, tensor_found, neural_found = convert_to_theano_var(obj[key])<EOL>normal_map[key] = normal_var<EOL>if tensor_found: theano_var_found = True<EOL>if neural_found: neural_var_found = True<EOL><DEDENT>return MapDict(normal_map), theano_var_found, neural_var_found<EOL><DEDENT>elif type(obj) == NeuralVariable:<EOL><INDENT>theano_tensor = obj.tensor<EOL>theano_tensor.tag.last_dim = obj.dim()<EOL>return theano_tensor, False, True<EOL><DEDENT>elif type(obj) == TensorVariable:<EOL><INDENT>return obj, True, False<EOL><DEDENT>elif type(obj) == slice:<EOL><INDENT>normal_args = []<EOL>theano_var_found = False<EOL>neural_var_found = False<EOL>for arg in [obj.start, obj.stop, obj.step]:<EOL><INDENT>normal_var, tensor_found, neural_found = convert_to_theano_var(arg)<EOL>normal_args.append(normal_var)<EOL>if tensor_found: theano_var_found = True<EOL>if neural_found: neural_var_found = True<EOL><DEDENT>return slice(*normal_args), theano_var_found, neural_var_found<EOL><DEDENT>else:<EOL><INDENT>return obj, False, False<EOL><DEDENT>

Convert neural vars to theano vars.
:param obj: NeuralVariable or list or dict or tuple
:return: theano var, test var, tensor found, neural var found


from theano.tensor.var import TensorVariable<EOL>from deepy.core.neural_var import NeuralVariable<EOL>if type(obj) == list:<EOL><INDENT>return [convert_to_neural_var(item) for item in obj]<EOL><DEDENT>elif type(obj) == tuple:<EOL><INDENT>return tuple(convert_to_neural_var(list(obj)))<EOL><DEDENT>elif type(obj) == dict:<EOL><INDENT>merged_map = {}<EOL>for key in obj:<EOL><INDENT>merged_map[key] = convert_to_neural_var(obj[key])<EOL><DEDENT>return merged_map<EOL><DEDENT>elif type(obj) == MapDict:<EOL><INDENT>merged_map = {}<EOL>for key in obj:<EOL><INDENT>merged_map[key] = convert_to_neural_var(obj[key])<EOL><DEDENT>return MapDict(merged_map)<EOL><DEDENT>elif type(obj) == TensorVariable:<EOL><INDENT>deepy_var = NeuralVariable(obj)<EOL>if hasattr(obj, '<STR_LIT>') and hasattr(obj.tag, '<STR_LIT>'):<EOL><INDENT>deepy_var.output_dim = obj.tag.last_dim<EOL><DEDENT>return deepy_var<EOL><DEDENT>else:<EOL><INDENT>return obj<EOL><DEDENT>

Convert object and a test object into neural var.
:param obj: tensor or list or dict or tuple
:param test_obj: NeuralVar or list or dict or tuple
:return:


def wrapper(*args, **kwargs):<EOL><INDENT>normal_args, tensor_found_in_args, neural_found_in_args = convert_to_theano_var(args)<EOL>normal_kwargs, tensor_found_in_kwargs, neural_found_in_kwargs = convert_to_theano_var(kwargs)<EOL>tensor_found = tensor_found_in_args or tensor_found_in_kwargs<EOL>neural_found = neural_found_in_args or neural_found_in_kwargs<EOL>if tensor_found and neural_found:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>normal_result = original_func(*normal_args, **normal_kwargs)<EOL>if tensor_found or (not neural_found and prefer_tensor):<EOL><INDENT>return normal_result<EOL><DEDENT>else:<EOL><INDENT>result_var = convert_to_neural_var(normal_result)<EOL>if (isinstance(normal_result, TensorVariable) and<EOL>hasattr(normal_result.tag, "<STR_LIT>") and<EOL>hasattr(normal_result.tag.test_value, "<STR_LIT>") and<EOL>normal_result.tag.test_value.shape):<EOL><INDENT>result_var.output_dim = normal_result.tag.test_value.shape[-<NUM_LIT:1>]<EOL><DEDENT>return result_var<EOL><DEDENT><DEDENT>return wrapper<EOL>

An annotation to enable theano-based fucntions to be called with NeuralVar.
:param original_func:
:param prefer_tensor: a switch to return tensors when no inputs
:return:


@neural_computation<EOL><INDENT>def iftrain(self, then_branch, else_branch):<DEDENT>

return ifelse(self._training_flag, then_branch, else_branch, name="<STR_LIT>")<EOL>


if self._is_training == flag: return<EOL>self._is_training = flag<EOL>if flag:<EOL><INDENT>self._training_flag.set_value(<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>self._training_flag.set_value(<NUM_LIT:0>)<EOL><DEDENT>

Switch training mode.
:param flag: switch on training mode when flag is True.


def __init__(self, input_dim=<NUM_LIT:0>, model=None, input_tensor=None, monitors=None,<EOL>cost=None, output=None, outputs=None, blocks=None, input_vars=None, target_vars=None):

from deepy.core.neural_var import NeuralVariable<EOL>from deepy.core.tensor_conversion import convert_to_theano_var<EOL>from theano.sandbox.cuda import CudaNdarraySharedVariable<EOL>super(ComputationalGraph, self).__init__(input_dim, input_tensor=input_tensor)<EOL>if model:<EOL><INDENT>self.stack(model)<EOL><DEDENT>if cost:<EOL><INDENT>self.stack(cost)<EOL><DEDENT>if output:<EOL><INDENT>if cost:<EOL><INDENT>self._test_output = output.tensor<EOL><DEDENT>else:<EOL><INDENT>self.stack(output)<EOL><DEDENT><DEDENT>if blocks:<EOL><INDENT>self.register(*blocks)<EOL><DEDENT>if input_vars:<EOL><INDENT>self.input_variables = [t.tensor for t in input_vars]<EOL><DEDENT>if target_vars:<EOL><INDENT>self.target_variables = [t.tensor for t in target_vars]<EOL><DEDENT>if outputs:<EOL><INDENT>if not output and not cost:<EOL><INDENT>self._test_output = None<EOL><DEDENT>self._test_outputs, _, _ = convert_to_theano_var(outputs)<EOL><DEDENT>if monitors:<EOL><INDENT>if type(monitors) == dict:<EOL><INDENT>monitors = monitors.items()<EOL><DEDENT>for monitor in monitors:<EOL><INDENT>if type(monitor) != tuple:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>name, var = monitor<EOL>if isinstance(var, NeuralVariable):<EOL><INDENT>var = var.tensor<EOL><DEDENT>if isinstance(var, CudaNdarraySharedVariable):<EOL><INDENT>var *= <NUM_LIT:1.0>  <EOL><DEDENT>self.training_monitors.append((name, var))<EOL>self.testing_monitors.append((name, var))<EOL><DEDENT><DEDENT>

Create a basic network.

Parameters:
    input_dim - dimension of input variable
    model - a short hand to specify the model
    config - network configuration
    input_tensor - specify the tensor of input if it's special


if seed != self.DEFAULT_SEED:<EOL><INDENT>self._seed = seed<EOL><DEDENT>elif '<STR_LIT>' in os.environ:<EOL><INDENT>self._seed = int(os.environ['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>self._seed = self.DEFAULT_SEED<EOL><DEDENT>if self._seed != self.DEFAULT_SEED:<EOL><INDENT>logging.info("<STR_LIT>" % self._seed)<EOL><DEDENT>self._numpy_rand = np.random.RandomState(seed=self._seed)<EOL>self._theano_rand = RandomStreams(seed=self._seed)<EOL>self._shared_rand = SharedRandomStreams(seed=self._seed)<EOL>self._default_initializer = None<EOL>

Initialize seed and global random variables.


def __init__(self, sequences=None, outputs=None, non_sequences=None, block=None, **kwargs):

self._sequences = sequences if sequences else {}<EOL>self._outputs = outputs if outputs else {}<EOL>self._non_sequences = non_sequences if non_sequences else {}<EOL>if not isinstance(self._sequences, dict) or not isinstance(self._outputs, dict) != dict or not isinstance(self._non_sequences, dict):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self._block = block<EOL>self._kwargs = kwargs<EOL>self._loop_vars = None<EOL>self._dummy_nodes = None<EOL>self._scan_local_vars = None<EOL>self._ordered_out_keys = []<EOL>self._scan_outputs = None<EOL>

A loop function to support "with" grammar.


from theano.tensor.var import TensorVariable<EOL>from deepy.core.neural_var import NeuralVariable<EOL>if not self._loop_vars:<EOL><INDENT>self._ordered_out_keys = self._outputs.keys()<EOL>seq_keys = self._sequences.keys()<EOL>filled_out_keys = [k for k in self._ordered_out_keys if self._outputs[k]]<EOL>nonseq_keys = self._non_sequences.keys()<EOL>dummy_tensors, self._scan_local_vars = get_dummy_args(<EOL>sequences=[self._sequences[k].tensor for k in seq_keys],<EOL>outputs_info=[self._outputs[k].tensor for k in self._ordered_out_keys],<EOL>non_sequences=[self._non_sequences[k].tensor for k in nonseq_keys],<EOL>**self._kwargs<EOL>)<EOL>dummy_map = dict(zip(seq_keys + filled_out_keys + nonseq_keys, dummy_tensors))<EOL>arg_map = self._sequences.copy()<EOL>arg_map.update(self._outputs)<EOL>arg_map.update(self._non_sequences)<EOL>self._loop_vars = LoopVars()<EOL>for k, dummy_tensor in dummy_map.items():<EOL><INDENT>dummy_var = NeuralVariable(dummy_tensor, dim=arg_map[k].dim())<EOL>self._loop_vars[k] = dummy_var<EOL><DEDENT><DEDENT>


from neural_var import NeuralVariable<EOL>if not self._loop_vars:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>replace_map = {}<EOL>for k, var in vars.items():<EOL><INDENT>if var is not None:<EOL><INDENT>replace_map[self._dummy_nodes[k].tensor] = var.tensor<EOL><DEDENT><DEDENT>outputs = {}<EOL>for k in self._outputs:<EOL><INDENT>if k not in self._loop_vars:<EOL><INDENT>raise Exception("<STR_LIT>".format(k))<EOL><DEDENT>output_node = theano.clone(self._loop_vars[k].tensor, replace_map)<EOL>outputs[k] = NeuralVariable(output_node, self._loop_vars[k].dim())<EOL><DEDENT>return outputs<EOL>

Internal scan with dummy input variables.


if not self._scan_outputs:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>return self._scan_outputs<EOL><DEDENT>

Get the output of the loop.
:rtype: MapDict


if args:<EOL><INDENT>output_vars = map(self._scan_outputs.get, args)<EOL>if len(output_vars) == <NUM_LIT:1>:<EOL><INDENT>return output_vars[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>return output_vars<EOL><DEDENT><DEDENT>else:<EOL><INDENT>return self._scan_outputs<EOL><DEDENT>

Get the outputs of the loop.
Return specific variables by passing the keys to the arguments.
:rtype: MapDict