def query(self, search_model: QueryModel):

query_parsed = query_parser(search_model.query)<EOL>self.logger.debug(f'<STR_LIT>')<EOL>if search_model.sort is not None:<EOL><INDENT>self._mapping_to_sort(search_model.sort.keys())<EOL>sort = self._sort_object(search_model.sort)<EOL><DEDENT>else:<EOL><INDENT>sort = [{"<STR_LIT>": "<STR_LIT>"}]<EOL><DEDENT>if search_model.query == {}:<EOL><INDENT>query = {'<STR_LIT>': {}}<EOL><DEDENT>else:<EOL><INDENT>query = query_parsed[<NUM_LIT:0>]<EOL><DEDENT>body = {<EOL>'<STR_LIT>': query,<EOL>'<STR_LIT>': sort,<EOL>'<STR_LIT>': (search_model.page - <NUM_LIT:1>) * search_model.offset,<EOL>'<STR_LIT:size>': search_model.offset,<EOL>}<EOL>page = self.driver._es.search(<EOL>index=self.driver._index,<EOL>doc_type='<STR_LIT>',<EOL>body=body,<EOL>q=query_parsed[<NUM_LIT:1>]<EOL>)<EOL>object_list = []<EOL>for x in page['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>object_list.append(x['<STR_LIT>'])<EOL><DEDENT>return object_list<EOL>

Query elasticsearch for objects.
        :param search_model: object of QueryModel.
        :return: list of objects that match the query.


return '<STR_LIT>'<EOL>

str: the type of this plugin (``'Elasticsearch'``)


def text_query(self, search_model: FullTextModel):

self.logger.debug('<STR_LIT>'.format(search_model.text))<EOL>if search_model.sort is not None:<EOL><INDENT>self._mapping_to_sort(search_model.sort.keys())<EOL>sort = self._sort_object(search_model.sort)<EOL><DEDENT>else:<EOL><INDENT>sort = [{"<STR_LIT>": "<STR_LIT>"}]<EOL><DEDENT>body = {<EOL>'<STR_LIT>': sort,<EOL>'<STR_LIT>': (search_model.page - <NUM_LIT:1>) * search_model.offset,<EOL>'<STR_LIT:size>': search_model.offset,<EOL>}<EOL>page = self.driver._es.search(<EOL>index=self.driver._index,<EOL>doc_type='<STR_LIT>',<EOL>body=body,<EOL>q=search_model.text<EOL>)<EOL>object_list = []<EOL>for x in page['<STR_LIT>']['<STR_LIT>']:<EOL><INDENT>object_list.append(x['<STR_LIT>'])<EOL><DEDENT>return object_list<EOL>

Query elasticsearch for objects.
        :param search_model: object of FullTextModel
        :return: list of objects that match the query.


self.logger.debug('<STR_LIT>'.format(resource_id))<EOL>return self.driver._es.get(<EOL>index=self.driver._index,<EOL>id=resource_id,<EOL>doc_type='<STR_LIT>'<EOL>)['<STR_LIT>']<EOL>

Read object in elasticsearch using the resource_id.
        :param resource_id: id of the object to be read.
        :return: object value from elasticsearch.


output = Cnf()<EOL>for x in cnf.dis:<EOL><INDENT>dont_add = False<EOL>for y in x:<EOL><INDENT>for z in x:<EOL><INDENT>if z == -y:<EOL><INDENT>dont_add = True<EOL>break<EOL><DEDENT><DEDENT>if dont_add: break<EOL><DEDENT>if dont_add: continue<EOL>if x not in output.dis:<EOL><INDENT>output.dis |= frozenset([x])<EOL><DEDENT><DEDENT>return output<EOL>

I just found a remarkably large bug in my SAT solver and found an
interesting solution.
Remove all b | -b
(-b | b) & (b | -a) & (-b | a) & (a | -a)
becomes
(b | -a) & (-b | a)

Remove all (-e) & (-e)
(-e | a) & (-e | a) & (-e | a) & (-e | a)
becomes
(-e | a)
(-b | b | c) becomes nothing, not (c)


err = _Bin(self.transit, self.limbdark, self.settings, self.arrays)<EOL>if err != _ERR_NONE: RaiseError(err)<EOL>

Bins the light curve model to the provided time array


ecc = self.ecc if not np.isnan(self.ecc) else np.sqrt(self.ecw**<NUM_LIT:2> + self.esw**<NUM_LIT:2>)<EOL>esw = self.esw if not np.isnan(self.esw) else ecc * np.sin(self.w)<EOL>aRs = ((G * self.rhos * (<NUM_LIT:1.> + self.MpMs) * <EOL>(self.per * DAYSEC)**<NUM_LIT>) / (<NUM_LIT> * np.pi))**(<NUM_LIT:1.>/<NUM_LIT>)<EOL>inc = np.arccos(self.bcirc/aRs)<EOL>becc = self.bcirc * (<NUM_LIT:1> - ecc**<NUM_LIT:2>)/(<NUM_LIT:1> - esw)<EOL>tdur = self.per / <NUM_LIT> / np.pi * np.arcsin(((<NUM_LIT:1.> + self.RpRs)**<NUM_LIT:2> -<EOL>becc**<NUM_LIT:2>)**<NUM_LIT:0.5> / (np.sin(inc) * aRs))<EOL>tdur *= np.sqrt(<NUM_LIT:1.> - ecc**<NUM_LIT>)/(<NUM_LIT:1.> - esw)<EOL>return tdur<EOL>

The approximate transit duration for the general case of an eccentric orbit


self.Free()<EOL>

Free the C arrays when the last reference to the class goes out of scope!


if self.arrays._calloc:<EOL><INDENT>_dbl_free(self.arrays._time)<EOL>_dbl_free(self.arrays._flux)<EOL>_dbl_free(self.arrays._bflx)<EOL>_dbl_free(self.arrays._M)<EOL>_dbl_free(self.arrays._E)<EOL>_dbl_free(self.arrays._f)<EOL>_dbl_free(self.arrays._r)<EOL>_dbl_free(self.arrays._x)<EOL>_dbl_free(self.arrays._y)<EOL>_dbl_free(self.arrays._z)<EOL>self.arrays._calloc = <NUM_LIT:0><EOL><DEDENT>if self.arrays._balloc:  <EOL><INDENT>_dbl_free(self.arrays._b)<EOL>self.arrays._balloc = <NUM_LIT:0><EOL><DEDENT>if self.arrays._ialloc:<EOL><INDENT>_dbl_free(self.arrays._iarr)<EOL>self.arrays._ialloc = <NUM_LIT:0><EOL><DEDENT>

Frees the memory used by all of the dynamically allocated C arrays.


if name is None:<EOL><INDENT>name = '<STR_LIT>'<EOL><DEDENT>attrs = {}<EOL>for field_name, field_info in six.iteritems(data_struct_dict):<EOL><INDENT>field_type = field_info['<STR_LIT:type>']<EOL>if field_type == DictField.FIELD_TYPE_NAME and isinstance(field_info.get('<STR_LIT>'), dict):<EOL><INDENT>field_info['<STR_LIT>'] = create_validator(field_info['<STR_LIT>'])<EOL><DEDENT>attrs[field_name] = create_field(field_info)<EOL><DEDENT>name = force_str(name)<EOL>return type(name, (Validator, ), attrs)<EOL>

create a Validator instance from data_struct_dict

:param data_struct_dict: a dict describe validator's fields, like the dict `to_dict()` method returned.
:param name: name of Validator class

:return: Validator instance


assert isinstance(raw_data, dict), '<STR_LIT>'.format(type(raw_data).__name__)<EOL>self.raw_data = raw_data<EOL>self.validated_data = None<EOL>self.errors = {}<EOL>


d = dict()<EOL>for name, field in six.iteritems(cls._FIELDS_MAP):<EOL><INDENT>field_info = field.to_dict()<EOL>d[name] = field_info<EOL><DEDENT>return d<EOL>


return data<EOL>

model-level validate.
sub-class can override this method to validate data, return modified data


params = list(cls.PARAMS)<EOL>bases = cls.__bases__<EOL>for base in bases:<EOL><INDENT>if issubclass(base, BaseField):<EOL><INDENT>params.extend(base._get_all_params())<EOL><DEDENT><DEDENT>return params<EOL>

Collect all PARAMS from this class and its parent class.


value = self._validate(value)<EOL>for v in self.validators:<EOL><INDENT>v(value)<EOL><DEDENT>return value<EOL>

return validated value or raise FieldValidationError.


service_url = self._service_locator.get_service_url(service, version)<EOL>return self.__copy_and_set('<STR_LIT>', self.__strip_trailing_slashes(service_url))<EOL>

Sets the service name and version the request should target

        Args:
            service (str): The name of the service as displayed in the services.json file
            version (str): The version of the service as displayed in the services.json file

        Returns:
            The request builder instance in order to chain calls


return self.__send('<STR_LIT>')<EOL>

Sends the request as parametrized with the DELETE verb

        Returns:
            The response object or body depending of the parametrization

        Raises:
            Any exception parametrized with the `throw` method


return self.__send('<STR_LIT:GET>')<EOL>

Sends the request as parametrized with the GET verb

        Returns:
            The response object or body depending of the parametrization

        Raises:
            Any exception parametrized with the `throw` method


return self.__send('<STR_LIT:POST>')<EOL>

Sends the request as parametrized with the POST verb

        Returns:
            The response object or body depending of the parametrization

        Raises:
            Any exception parametrized with the `throw` method


return self.__send('<STR_LIT>')<EOL>

Sends the request as parametrized with the PUT verb

        Returns:
            The response object or body depending of the parametrization

        Raises:
            Any exception parametrized with the `throw` method


return self.__copy_and_set('<STR_LIT:url>', url)<EOL>

Sets the request target url

        Args:
            url (str): The url the request should be targeted to

        Returns:
            The request builder instance in order to chain calls


return self.__copy_and_set('<STR_LIT>', self.__strip_leading_slashes(endpoint))<EOL>

Sets the endpoint of the service the request should target

        Args:
            endpoint (str): The endpoint that will be concatenated to the service url

        Returns:
            The request builder instance in order to chain calls


@classmethod<EOL><INDENT>def new(cls, access_token, environment='<STR_LIT>'):<DEDENT>

return cls(<EOL>storage_client=StorageClient.new(access_token, environment=environment))<EOL>


self.__validate_storage_path(path)<EOL>entity = self.api_client.get_entity_by_query(path=path)<EOL>if entity['<STR_LIT>'] not in self.__BROWSABLE_TYPES:<EOL><INDENT>raise StorageArgumentException('<STR_LIT>'<EOL>'<STR_LIT>'.format(entity['<STR_LIT>']))<EOL><DEDENT>entity_uuid = entity['<STR_LIT>']<EOL>file_names = []<EOL>more_pages = True<EOL>page_number = <NUM_LIT:1><EOL>while more_pages:<EOL><INDENT>response = self.api_client.list_folder_content(<EOL>entity_uuid, page=page_number, ordering='<STR_LIT:name>')<EOL>more_pages = response['<STR_LIT>'] is not None<EOL>page_number += <NUM_LIT:1><EOL>for child in response['<STR_LIT>']:<EOL><INDENT>pattern = '<STR_LIT>' if child['<STR_LIT>'] == '<STR_LIT>' else '<STR_LIT>'<EOL>file_names.append(pattern.format(name=child['<STR_LIT:name>']))<EOL><DEDENT><DEDENT>return file_names<EOL>

List the entities found directly under the given path.

        Args:
            path (str): The path of the entity to be listed. Must start with a '/'.

        Returns:
            The list of entity names directly under the given path:

                u'/12345/folder_1'

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


self.__services_url = services_url<EOL>

Create new service locator
            Arguments:
                services_url: The url of the json file where the services url are defined
            Returns:
                A service locator instance


def get_service_url(self, service, version):

return self.__get_services()[service][version]<EOL>

Get the service URL
            Arguments:
                service: The service name
                version: The service version
            Returns:
                The URL where the service is located


Delete a file.

        Args:
            file_id (str): The UUID of the file to delete.

        Returns:
            None

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(folder):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(folder))<EOL><DEDENT>self._authenticated_request.to_endpoint('<STR_LIT>'.format(folder)).delete()<EOL>

Delete a folder. It will recursively delete all the content.

        Args:
            folder_id (str): The UUID of the folder to be deleted.

        Returns:
            None

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: 403
            StorageNotFoundException: 404
            HTTPError: other non-20x error codes


return self._authenticated_request.to_endpoint('<STR_LIT>').with_json_body(self._prep_params(locals())).return_body().post()<EOL>

Create a new project.

        Args:
            collab_id (int): The id of the collab the project should be created in.

        Returns:
            A dictionary of details of the created project::

                {
                    u'collab_id': 12998,
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-21T14:06:32.293902Z',
                    u'description': u'',
                    u'entity_type': u'project',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-21T14:06:32.293967Z',
                    u'name': u'12998',
                    u'uuid': u'2516442e-1e26-4de1-8ed8-94523224cc40'
                }

        Raises:
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_id)).return_body().get()["<STR_LIT>"]<EOL>

Retrieve entity Collab ID.

        Args:
            entity_id (str): The UUID of the requested entity.

        Returns:
            The id as interger of the Collebaration to which the entity belongs

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def download_file_content(self, file_id, etag=None):

if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>headers = {'<STR_LIT>': '<STR_LIT>'}<EOL>if etag:<EOL><INDENT>headers['<STR_LIT>'] = etag<EOL><DEDENT>resp = self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).with_headers(headers).get()<EOL>if resp.status_code == <NUM_LIT>:<EOL><INDENT>return (None, None)<EOL><DEDENT>if '<STR_LIT>' not in resp.headers:<EOL><INDENT>raise StorageException('<STR_LIT>')<EOL><DEDENT>return (resp.headers['<STR_LIT>'], resp.content)<EOL>

Download file content.

        Args:
            file_id (str): The UUID of the file whose content is requested
            etag (str): If the content is not changed since the provided ETag,
                the content won't be downloaded. If the content is changed, it
                will be downloaded and returned with its new ETag.

        Note:
            ETags should be enclosed in double quotes::

                my_etag = '"71e1ed9ee52e565a56aec66bc648a32c"'

        Returns:
            A tuple of ETag and content (etag, content) if the content was
            retrieved. If an etag was provided, and content didn't change
            returns (None, None)::

                ('"71e1ed9ee52e565a56aec66bc648a32c"', 'Hello world!')

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


return {k: v for (k, v) in params.items() if v is not None and k != '<STR_LIT>'}<EOL>

Remove empty (None) valued keywords and self from function parameters


def get_metadata(self, entity_type, entity_id):

if not is_valid_uuid(entity_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(entity_id))<EOL><DEDENT>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(entity_type, entity_id)).return_body().get()<EOL>

Get metadata of an entity.

        Args:
            entity_type (str): Type of the entity. Admitted values: ['project',
                'folder', 'file'].
            entity_id (str): The UUID of the entity to be modified.

        Returns:
            A dictionary of the metadata::

                {
                    u'bar': u'200',
                    u'foo': u'100'
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


def copy_file_content(self, file_id, source_file):

if not is_valid_uuid(file_id):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(file_id))<EOL><DEDENT>if not is_valid_uuid(source_file):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(source_file))<EOL><DEDENT>self._authenticated_request.to_endpoint('<STR_LIT>'.format(file_id)).with_headers({'<STR_LIT>': source_file}).put()<EOL>

Copy file content from source file to target file.

        Args:
            file_id (str): The UUID of the file whose content is written.
            source_file (str): The UUID of the file whose content is copied.

        Returns:
            None

        Raises:
        StorageArgumentException: Invalid arguments
        StorageForbiddenException: Server response code 403
        StorageNotFoundException: Server response code 404
        StorageException: other 400-600 error codes


def list_folder_content(self, folder, name=None, entity_type=None,<EOL>content_type=None, page_size=DEFAULT_PAGE_SIZE,<EOL>page=None, ordering=None):

if not is_valid_uuid(folder):<EOL><INDENT>raise StorageArgumentException(<EOL>'<STR_LIT>'.format(folder))<EOL><DEDENT>params = self._prep_params(locals())<EOL>del params['<STR_LIT>']  <EOL>return self._authenticated_request.to_endpoint('<STR_LIT>'.format(folder)).with_params(params).return_body().get()<EOL>

List files and folders (not recursively) contained in the folder.

        This function does not retrieve all results, pages have
        to be manually retrieved by the caller.

        Args:
            folder (str): The UUID of the requested folder.
            name (str): Optional filter on entity name.
            entity_type (str): Optional filter on entity type.
                Admitted values: ['file', 'folder'].
            content_type (str): Optional filter on entity content type (only
                files are returned).
            page_size (int): Number of elements per page.
            page (int): Number of the page.
            ordering (str): Indicate on which fields to sort the result. Prepend
                '-' to invert order. Multiple values can be provided.
                Ordering is supported on: ['name', 'created_on', 'modified_on'].
                Example: 'ordering=name,created_on'

        Returns:
            A dictionary of the results::

                {
                u'count': 1,
                u'next': None,
                u'previous': None,
                u'results': [{u'content_type': u'plain/text',
                    u'created_by': u'303447',
                    u'created_on': u'2017-03-13T10:17:01.688472Z',
                    u'description': u'',
                    u'entity_type': u'file',
                    u'modified_by': u'303447',
                    u'modified_on': u'2017-03-13T10:17:01.688632Z',
                    u'name': u'file_1',
                    u'parent': u'eac11058-4ae0-4ea9-ada8-d3ea23887509',
                    u'uuid': u'0e17eaac-cb00-4336-b9d7-657026844281'}]
                }

        Raises:
            StorageArgumentException: Invalid arguments
            StorageForbiddenException: Server response code 403
            StorageNotFoundException: Server response code 404
            StorageException: other 400-600 error codes


try:<EOL><INDENT>stdout = completed_cmd.stdout.encode('<STR_LIT:utf-8>').decode()<EOL><DEDENT>except AttributeError:<EOL><INDENT>try:<EOL><INDENT>stdout = str(bytes(completed_cmd.stdout), '<STR_LIT>').strip()<EOL><DEDENT>except AttributeError:<EOL><INDENT>stdout = str(bytes(completed_cmd.stdout).decode('<STR_LIT:utf-8>')).strip()<EOL><DEDENT><DEDENT>try:<EOL><INDENT>stderr = completed_cmd.stderr.encode('<STR_LIT:utf-8>').decode()<EOL><DEDENT>except AttributeError:<EOL><INDENT>try:<EOL><INDENT>stderr = str(bytes(completed_cmd.stderr), '<STR_LIT>').strip()<EOL><DEDENT>except AttributeError:<EOL><INDENT>stderr = str(bytes(completed_cmd.stderr).decode('<STR_LIT:utf-8>')).strip()<EOL><DEDENT><DEDENT>return ParsedCompletedCommand(<EOL>completed_cmd.returncode,<EOL>completed_cmd.args,<EOL>stdout,<EOL>stderr<EOL>)<EOL>


exp_cache_file = self.get_data_path(self.name, self.version)<EOL>if not os.path.isdir(exp_cache_file):<EOL><INDENT>os.makedirs(exp_cache_file)<EOL><DEDENT>

Inits a file that we log historical experiments
:return:


params = []<EOL>for k in trial.__dict__:<EOL><INDENT>v = trial.__dict__[k]<EOL>if v is None or v == False:<EOL><INDENT>continue<EOL><DEDENT>if self.__should_escape(v):<EOL><INDENT>cmd = '<STR_LIT>'.format(k, v)<EOL><DEDENT>else:<EOL><INDENT>cmd = '<STR_LIT>'.format(k, v)<EOL><DEDENT>params.append(cmd)<EOL><DEDENT>params.append('<STR_LIT>'.format(HyperOptArgumentParser.TRIGGER_CMD))<EOL>full_cmd = '<STR_LIT:U+0020>'.join(params)<EOL>return full_cmd<EOL>

Turns hopt trial into script params
:param trial:
:return:


def run_setup(script_name, script_args=None, stop_after="<STR_LIT>"):

if stop_after not in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>raise ValueError("<STR_LIT>" % stop_after)<EOL><DEDENT>core._setup_stop_after = stop_after<EOL>save_argv = sys.argv<EOL>glocals = copy(globals())<EOL>glocals['<STR_LIT>'] = script_name<EOL>glocals['<STR_LIT>'] = "<STR_LIT:__main__>"<EOL>try:<EOL><INDENT>try:<EOL><INDENT>sys.argv[<NUM_LIT:0>] = script_name<EOL>if script_args is not None:<EOL><INDENT>sys.argv[<NUM_LIT:1>:] = script_args<EOL><DEDENT>f = open(script_name)<EOL>try:<EOL><INDENT>exec(f.read(), glocals, glocals)<EOL><DEDENT>finally:<EOL><INDENT>f.close()<EOL><DEDENT><DEDENT>finally:<EOL><INDENT>sys.argv = save_argv<EOL>core._setup_stop_after = None<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>logging.warn("<STR_LIT>", exc_info=True)<EOL><DEDENT>if core._setup_distribution is None:<EOL><INDENT>raise RuntimeError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>script_name)<EOL><DEDENT>return core._setup_distribution<EOL>

Run a setup script in a somewhat controlled environment, and
    return the Distribution instance that drives things.  This is useful
    if you need to find out the distribution meta-data (passed as
    keyword args from 'script' to 'setup()', or the contents of the
    config files or command-line.

    'script_name' is a file that will be run with 'execfile()';
    'sys.argv[0]' will be replaced with 'script' for the duration of the
    call.  'script_args' is a list of strings; if supplied,
    'sys.argv[1:]' will be replaced by 'script_args' for the duration of
    the call.

    'stop_after' tells 'setup()' when to stop processing; possible
    values:
      init
        stop after the Distribution instance has been created and
        populated with the keyword arguments to 'setup()'
      config
        stop after config files have been parsed (and their data
        stored in the Distribution instance)
      commandline
        stop after the command-line ('sys.argv[1:]' or 'script_args')
        have been parsed (and the data stored in the Distribution)
      run [default]
        stop after all commands have been run (the same as if 'setup()'
        had been called in the usual way

    Returns the Distribution instance, which provides all information
    used to drive the Distutils.


train_set = self.train_set()<EOL>if isinstance(train_set, collections.Iterable):<EOL><INDENT>return len(list(train_set))<EOL><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT>

Return size of training data. (optional)
:rtype: number


def pad_dataset(subset, side="<STR_LIT:right>", length=-<NUM_LIT:1>):

assert length == -<NUM_LIT:1> or length > <NUM_LIT:0><EOL>if type(subset[<NUM_LIT:0>][<NUM_LIT:0>][<NUM_LIT:0>]) in [float, int, np.int64, np.int32, np.float32]:<EOL><INDENT>return _pad_2d(subset, side, length)<EOL><DEDENT>else:<EOL><INDENT>return _pad_3d(subset, side, length)<EOL><DEDENT>

Pad data set to specified length.
Parameters:
    length - max length, a just to the max length in the batch if length is -1


for layer, hidden in zip(self.layers, self._hidden_outputs):<EOL><INDENT>self.training_monitors.append(('<STR_LIT>' % (layer.name), abs(hidden).mean()))<EOL><DEDENT>

Monitoring the outputs of each layer.
Useful for troubleshooting convergence problems.


def load_params(self, path, exclude_free_params=False):

if not os.path.exists(path): return;<EOL>logging.info("<STR_LIT>" % path)<EOL>if exclude_free_params:<EOL><INDENT>params_to_load = self.parameters<EOL><DEDENT>else:<EOL><INDENT>params_to_load = self.all_parameters<EOL><DEDENT>if path.endswith("<STR_LIT>"):<EOL><INDENT>opener = gzip.open if path.lower().endswith('<STR_LIT>') else open<EOL>handle = opener(path, '<STR_LIT:rb>')<EOL>saved_params = pickle.load(handle)<EOL>handle.close()<EOL>for target, source in zip(params_to_load, saved_params):<EOL><INDENT>logging.info('<STR_LIT>', target.name, source.shape)<EOL>target.set_value(source)<EOL><DEDENT><DEDENT>elif path.endswith("<STR_LIT>"):<EOL><INDENT>arrs = np.load(path)<EOL>for target, idx in zip(params_to_load, range(len(arrs.keys()))):<EOL><INDENT>source = arrs['<STR_LIT>' % idx]<EOL>logging.info('<STR_LIT>', target.name, source.shape)<EOL>target.set_value(source)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>" % path)<EOL><DEDENT>self.train_logger.load(path)<EOL>


for layer in layers:<EOL><INDENT>self.register_layer(layer)<EOL><DEDENT>

Register multiple layers as the components of the network.
The parameter of those layers will be trained.
But the output of the layer will not be stacked.


for layer in layers:<EOL><INDENT>self.stack_layer(layer)<EOL><DEDENT>return self<EOL>


Register the layer so that it's param will be trained.
But the output of the layer will not be stacked.


return T.constant(<NUM_LIT:0>)<EOL>


logging.info("<STR_LIT>", "<STR_LIT:U+0020>".join(map(str, self.input_variables)))<EOL>logging.info("<STR_LIT>", "<STR_LIT:U+0020>".join(map(str, self.target_variables)))<EOL>logging.info("<STR_LIT>", "<STR_LIT:U+0020>".join(map(str, self.all_parameters)))<EOL>logging.info("<STR_LIT>", self.parameter_count)<EOL>


self.stack(*layers)<EOL>self.encoding_layes.extend(layers)<EOL>

Stack encoding layers, this must be done before stacking decoding layers.


self.stack(*layers)<EOL>self.decoding_layers.extend(layers)<EOL>


gs = [dim_to_var(p.ndim) for p in params]<EOL>updates, _ = optimize_updates(params, gs, config)<EOL>return theano.function(gs, [], updates=updates)<EOL>

Create a optimizing function receives gradients.
Parameters:
    params - parameters
    config - training configuration
Returns:
    updating function receives gradients


def __init__(self, patience=<NUM_LIT:3>, anneal_times=<NUM_LIT:4>):

self._iter = <NUM_LIT:0><EOL>self._annealed_iter = <NUM_LIT:0><EOL>self._patience = patience<EOL>self._anneal_times = anneal_times<EOL>self._annealed_times = <NUM_LIT:0><EOL>self._learning_rate = <NUM_LIT:0><EOL>if type(self._learning_rate) == float:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>


def momentum_core(params, gradients, momentum=<NUM_LIT>, learning_rate=<NUM_LIT>):

free_parameters = []<EOL>updates = []<EOL>for param, grad in zip(params, gradients):<EOL><INDENT>delta = learning_rate * grad<EOL>velocity = theano.shared(np.zeros_like(param.get_value()), name=param.name + '<STR_LIT>')<EOL>updates.append((velocity, momentum * velocity - delta))<EOL>updates.append((param, param + velocity))<EOL>free_parameters.append(velocity)<EOL><DEDENT>return updates, free_parameters<EOL>


def report(self, score_map, type="<STR_LIT>", epoch=-<NUM_LIT:1>, new_best=False):

type_str = type<EOL>if len(type_str) < <NUM_LIT:5>:<EOL><INDENT>type_str += "<STR_LIT:U+0020>" * (<NUM_LIT:5> - len(type_str))<EOL><DEDENT>info = "<STR_LIT:U+0020>".join("<STR_LIT>" % el for el in score_map.items())<EOL>current_epoch = epoch if epoch > <NUM_LIT:0> else self.current_epoch()<EOL>epoch_str = "<STR_LIT>".format(current_epoch + <NUM_LIT:1>)<EOL>if epoch < <NUM_LIT:0>:<EOL><INDENT>epoch_str = "<STR_LIT>"<EOL>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL><DEDENT>marker = "<STR_LIT>" if new_best else "<STR_LIT>"<EOL>message = "<STR_LIT>".format(type_str, epoch_str, info, marker)<EOL>self.network.train_logger.record(message)<EOL>logging.info(message)<EOL>

Report the scores and record them in the log.


def _run_train(self, epoch, train_set, train_size=None):

self.network.train_logger.record_epoch(epoch + <NUM_LIT:1>)<EOL>costs = self.train_step(train_set, train_size)<EOL>if not epoch % self.config.monitor_frequency:<EOL><INDENT>self.report(dict(costs), "<STR_LIT:train>", epoch)<EOL><DEDENT>self.last_run_costs = costs<EOL>return costs<EOL>


for controller in controllers:<EOL><INDENT>if isinstance(controller, TrainingController):<EOL><INDENT>controller.bind(self)<EOL><DEDENT>self._iter_controllers.append(controller)<EOL><DEDENT>

Add iteration callbacks function (receives an argument of the trainer).
:param controllers: can be a `TrainingController` or a function.
:type funcs: list of TrainingContoller


return self._epoch<EOL>


self._ended = True<EOL>


def __init__(self, network, config=None):

super(CustomizeTrainer, self).__init__(network, config)<EOL>

Basic neural network trainer.
:type network: deepy.NeuralNetwork
:type config: deepy.conf.TrainerConfig
:return:


return T.grad(self.cost, params)<EOL>


network_updates = list(self.network.updates) + list(self.network.training_updates)<EOL>learning_updates = list(self._learning_updates())<EOL>update_list = network_updates + learning_updates<EOL>logging.info("<STR_LIT>" % "<STR_LIT:U+0020>".join(map(str, [x[<NUM_LIT:0>] for x in network_updates])))<EOL>logging.info("<STR_LIT>" % "<STR_LIT:U+0020>".join(map(str, [x[<NUM_LIT:0>] for x in learning_updates])))<EOL>variables = self.network.input_variables + self.network.target_variables<EOL>givens = None<EOL>return theano.function(<EOL>variables,<EOL>map(lambda v: theano.Out(v, borrow=True), self.training_variables),<EOL>updates=update_list, allow_input_downcast=True,<EOL>mode=self.config.get("<STR_LIT>", None),<EOL>givens=givens)<EOL>

Get the learning function.
:param func:
:return:


self._counter += <NUM_LIT:1><EOL>if self._counter % self._freq == <NUM_LIT:0>:<EOL><INDENT>cnt = <NUM_LIT:0.><EOL>sum_map = defaultdict(float)<EOL>for x in self._trainer.get_data(self._data_split):<EOL><INDENT>val_map = self.run(x)<EOL>if not isinstance(val_map, dict):<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>for k, val in val_map.items():<EOL><INDENT>sum_map[k] += val<EOL><DEDENT>cnt += <NUM_LIT:1><EOL><DEDENT>for k in sum_map:<EOL><INDENT>sum_map[k] /= cnt<EOL><DEDENT>new_best = self.compare(sum_map)<EOL>self._trainer.report(sum_map, self._data_split, new_best=new_best)<EOL>if new_best:<EOL><INDENT>self._trainer.save_checkpoint(self._save_path)<EOL><DEDENT><DEDENT>

This function will be called after each iteration.


cri_val = cost_map[self._criteria]<EOL>if self._best_criteria is None:<EOL><INDENT>self._best_criteria = cri_val<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>if self._smaller_is_better and cri_val < self._best_criteria:<EOL><INDENT>self._best_criteria = cri_val<EOL>return True<EOL><DEDENT>elif not self._smaller_is_better and cri_val > self._best_criteria:<EOL><INDENT>self._best_criteria = cri_val<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>

Compare to previous records and return whether the given cost is a new best.
:return: True if the given cost is a new best


def compute_context_vector(self, prev_state, inputs, precomputed_values=None, mask=None):

precomputed_values = precomputed_values if precomputed_values else self.precompute(inputs)<EOL>align_weights = self.compute_alignments(prev_state, precomputed_values, mask)<EOL>context_vector = T.sum(align_weights[:, :, None] * inputs, axis=<NUM_LIT:1>)<EOL>return context_vector<EOL>

Compute the context vector with soft attention.


for key, node in monitors:<EOL><INDENT>if key not in self._registered_monitors:<EOL><INDENT>node *= <NUM_LIT:1.0> <EOL>self.training_monitors.append((key, node))<EOL>self.testing_monitors.append((key, node))<EOL>self._registered_monitors.add(key)<EOL><DEDENT><DEDENT>

Register monitors they should be tuple of name and Theano variable.


self.training_callbacks.extend(callbacks)<EOL>

Register callback for each iteration in the training.


self.external_inputs.extend(variables)<EOL>


for param in parameters:<EOL><INDENT>self.parameter_count += np.prod(param.get_value().shape)<EOL><DEDENT>self.parameters.extend(parameters)<EOL>


name = kwargs['<STR_LIT:name>'] if '<STR_LIT:name>' in kwargs else "<STR_LIT>".format(self._BLOCK_COUNT + <NUM_LIT:1>)<EOL>super(Block, self).__init__(name)<EOL>self._BLOCK_COUNT += <NUM_LIT:1><EOL>self.layers = list(layers)<EOL>self.fixed = False<EOL>

Create a new parameter block with some layers.
You can also specify a name through kwargs.


if self.fixed:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>self.layers.append(layer)<EOL>

Register one connected layer.
:type layer: NeuralLayer


for layer in layers:<EOL><INDENT>self.register_layer(layer)<EOL><DEDENT>

Register many connected layers.
:type layers: list of NeuralLayer


@neural_computation<EOL><INDENT>def get_initial_states(self, input_var, init_state=None):<DEDENT>

initial_states = {}<EOL>for state in self.state_names:<EOL><INDENT>if state != "<STR_LIT:state>" or not init_state:<EOL><INDENT>if self._input_type == '<STR_LIT>' and input_var.ndim == <NUM_LIT:2>:<EOL><INDENT>init_state = T.alloc(np.cast[env.FLOATX](<NUM_LIT:0.>), self.hidden_size)<EOL><DEDENT>else:<EOL><INDENT>init_state = T.alloc(np.cast[env.FLOATX](<NUM_LIT:0.>), input_var.shape[<NUM_LIT:0>], self.hidden_size)<EOL><DEDENT><DEDENT>initial_states[state] = init_state<EOL><DEDENT>return initial_states<EOL>


@neural_computation<EOL><INDENT>def get_step_inputs(self, input_var, states=None, mask=None, additional_inputs=None):<DEDENT>

step_inputs = {}<EOL>if self._input_type == "<STR_LIT>":<EOL><INDENT>if not additional_inputs:<EOL><INDENT>additional_inputs = []<EOL><DEDENT>if mask:<EOL><INDENT>step_inputs['<STR_LIT>'] = mask.dimshuffle(<NUM_LIT:1>, <NUM_LIT:0>)<EOL><DEDENT>step_inputs.update(self.merge_inputs(input_var, additional_inputs=additional_inputs))<EOL><DEDENT>else:<EOL><INDENT>if additional_inputs:<EOL><INDENT>step_inputs.update(self.merge_inputs(None, additional_inputs=additional_inputs))<EOL><DEDENT><DEDENT>if states:<EOL><INDENT>for name in self.state_names:<EOL><INDENT>step_inputs[name] = states[name]<EOL><DEDENT><DEDENT>return step_inputs<EOL>


if self._is_training == flag: return<EOL>self._is_training = flag<EOL>if flag:<EOL><INDENT>self._training_flag.set_value(<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>self._training_flag.set_value(<NUM_LIT:0>)<EOL><DEDENT>

Switch training mode.
:param flag: switch on training mode when flag is True.


@neural_computation<EOL><INDENT>def iftrain(self, then_branch, else_branch):<DEDENT>

return ifelse(self._training_flag, then_branch, else_branch, name="<STR_LIT>")<EOL>


if seed != self.DEFAULT_SEED:<EOL><INDENT>self._seed = seed<EOL><DEDENT>elif '<STR_LIT>' in os.environ:<EOL><INDENT>self._seed = int(os.environ['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>self._seed = self.DEFAULT_SEED<EOL><DEDENT>if self._seed != self.DEFAULT_SEED:<EOL><INDENT>logging.info("<STR_LIT>" % self._seed)<EOL><DEDENT>self._numpy_rand = np.random.RandomState(seed=self._seed)<EOL>self._theano_rand = RandomStreams(seed=self._seed)<EOL>self._shared_rand = SharedRandomStreams(seed=self._seed)<EOL>self._default_initializer = None<EOL>

Initialize seed and global random variables.


from neural_var import NeuralVariable<EOL>if not self._loop_vars:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>replace_map = {}<EOL>for k, var in vars.items():<EOL><INDENT>if var is not None:<EOL><INDENT>replace_map[self._dummy_nodes[k].tensor] = var.tensor<EOL><DEDENT><DEDENT>outputs = {}<EOL>for k in self._outputs:<EOL><INDENT>if k not in self._loop_vars:<EOL><INDENT>raise Exception("<STR_LIT>".format(k))<EOL><DEDENT>output_node = theano.clone(self._loop_vars[k].tensor, replace_map)<EOL>outputs[k] = NeuralVariable(output_node, self._loop_vars[k].dim())<EOL><DEDENT>return outputs<EOL>

Internal scan with dummy input variables.


return disconnected_grad(x)<EOL>

Disconnect a variable from backpropagation.


self.end_time = time.time()<EOL>


raise NotImplementedError<EOL>


def __init__(self, uniform=False, seed=None):

super(KaimingHeInitializer, self).__init__(seed)<EOL>self.uniform = uniform<EOL>

Parameters:
    uniform - uniform distribution, default Gaussian
    seed - random seed


for elt in iterable_to_pickle:<EOL><INDENT>StreamPickler.dump_one(elt, file_obj)<EOL><DEDENT>

dump contents of an iterable iterable_to_pickle to file_obj, a file
opened in write mode


cur_elt = []<EOL>for line in file_obj:<EOL><INDENT>cur_elt.append(line)<EOL>if line == '<STR_LIT:\n>':<EOL><INDENT>pickled_elt_str = '<STR_LIT>'.join(cur_elt)<EOL>cur_elt = []<EOL>try:<EOL><INDENT>elt = loads(pickled_elt_str)<EOL><DEDENT>except ValueError:<EOL><INDENT>continue<EOL><DEDENT>yield elt<EOL><DEDENT><DEDENT>

load contents from file_obj, returning a generator that yields one
element at a time


from activations import get_activation<EOL>return get_activation(method)(var)<EOL>

An activation function.
:param var: input var
:param method: type of activation, such as `relu`,`tanh`,`sigmoid`


def handle_control(self, req, worker_id, req_info):

if self.start_time is None: self.start_time = time.time()<EOL>response = "<STR_LIT>"<EOL>if req == '<STR_LIT>':<EOL><INDENT>if self.num_train_batches == <NUM_LIT:0>:<EOL><INDENT>response = "<STR_LIT>"<EOL><DEDENT>elif self._done:<EOL><INDENT>response = "<STR_LIT>"<EOL>self.worker_is_done(worker_id)<EOL><DEDENT>elif self._evaluating:<EOL><INDENT>response = '<STR_LIT>'<EOL><DEDENT>elif not self.batch_pool:<EOL><INDENT>if self._train_costs:<EOL><INDENT>with self._lock:<EOL><INDENT>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>mean_costs = []<EOL>for i in range(len(self._training_names)):<EOL><INDENT>mean_costs.append(np.mean([c[i] for c in self._train_costs]))<EOL><DEDENT>self.log("<STR_LIT>".format(<EOL>self.epoch,<EOL>self.get_monitor_string(zip(self._training_names, mean_costs)))<EOL>)<EOL><DEDENT><DEDENT>response = {'<STR_LIT>': None, '<STR_LIT>': self._best_valid_cost}<EOL>self._evaluating = True<EOL><DEDENT>else:<EOL><INDENT>if worker_id not in self.prepared_worker_pool:<EOL><INDENT>response = {"<STR_LIT>": self.feed_hyperparams()}<EOL>self.prepared_worker_pool.add(worker_id)<EOL><DEDENT>elif self._iters_from_last_valid >= self._valid_freq:<EOL><INDENT>response = {'<STR_LIT>': None, '<STR_LIT>': self._best_valid_cost}<EOL>self._iters_from_last_valid = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>response = {"<STR_LIT:train>": self.feed_batches()}<EOL><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>with self._lock:<EOL><INDENT>self._evaluating = False<EOL>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>if '<STR_LIT>' in req and req['<STR_LIT>']:<EOL><INDENT>self.log("<STR_LIT>".format(<EOL>self.epoch,<EOL>self.get_monitor_string(req['<STR_LIT>']),<EOL>worker_id)<EOL>)<EOL><DEDENT>if '<STR_LIT>' in req and req['<STR_LIT>']:<EOL><INDENT>valid_J = req['<STR_LIT>'][<NUM_LIT:0>][<NUM_LIT:1>]<EOL>if valid_J < self._best_valid_cost:<EOL><INDENT>self._best_valid_cost = valid_J<EOL>star_str = "<STR_LIT:*>"<EOL><DEDENT>else:<EOL><INDENT>star_str = "<STR_LIT>"<EOL>self.log("<STR_LIT>".format(<EOL>self.epoch,<EOL>self.get_monitor_string(req['<STR_LIT>']),<EOL>star_str,<EOL>worker_id))<EOL><DEDENT><DEDENT>continue_training = self.prepare_epoch()<EOL>self._epoch_start_time = time.time()<EOL>if not continue_training:<EOL><INDENT>self._done = True<EOL>self.log("<STR_LIT>".format(time.time() - self.start_time))<EOL>response = "<STR_LIT>"<EOL><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>with self._lock:<EOL><INDENT>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>if '<STR_LIT>' in req:<EOL><INDENT>valid_J = req['<STR_LIT>'][<NUM_LIT:0>][<NUM_LIT:1>]<EOL>if valid_J < self._best_valid_cost:<EOL><INDENT>self._best_valid_cost = valid_J<EOL>star_str = "<STR_LIT:*>"<EOL><DEDENT>else:<EOL><INDENT>star_str = "<STR_LIT>"<EOL><DEDENT>self.log("<STR_LIT>".format(<EOL>self.get_monitor_string(req['<STR_LIT>']),<EOL>star_str,<EOL>worker_id<EOL>))<EOL><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>costs = req['<STR_LIT>']<EOL>self._train_costs.append(costs)<EOL>sys.stdout.write("<STR_LIT>" % (<EOL>self._current_iter * <NUM_LIT:100> / self.num_train_batches,<EOL>costs[<NUM_LIT:0>], float(len(self._train_costs) * self.sync_freq) / (time.time() - self._epoch_start_time)))<EOL>sys.stdout.flush()<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>self.num_train_batches = req['<STR_LIT>']<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>response = self._easgd_alpha<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>response = {"<STR_LIT>": self.feed_hyperparams()}<EOL><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>with self._lock:<EOL><INDENT>sys.stdout.write("<STR_LIT:\r>")<EOL>sys.stdout.flush()<EOL>self.log("<STR_LIT>".format(worker_id))<EOL>if self.epoch == <NUM_LIT:0>:<EOL><INDENT>schedule_params = req['<STR_LIT>']<EOL>sch_str = "<STR_LIT:U+0020>".join("<STR_LIT>".format(a, b) for (a, b) in schedule_params.items())<EOL>self.log("<STR_LIT>".format(sch_str))<EOL>for key, val in schedule_params.items():<EOL><INDENT>if not val: continue<EOL>if key == '<STR_LIT>':<EOL><INDENT>self._lr = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self.epoch_start_halving = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self._halving_freq = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self.end_at = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self.sync_freq = val<EOL><DEDENT>elif key == '<STR_LIT>':<EOL><INDENT>self._valid_freq = val<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>elif '<STR_LIT>' in req:<EOL><INDENT>self._training_names = req['<STR_LIT>']<EOL>self._evaluation_names = req['<STR_LIT>']<EOL><DEDENT>return response<EOL>

Handles a control_request received from a worker.
Returns:
    string or dict: response

    'stop' - the worker should quit
    'wait' - wait for 1 second
    'eval' - evaluate on valid and test set to start a new epoch
    'sync_hyperparams' - set learning rate
    'valid' - evaluate on valid and test set, then save the params
    'train' - train next batches


object.__setattr__(self, "<STR_LIT>", {})<EOL>object.__setattr__(self, "<STR_LIT>", set())<EOL>object.__setattr__(self, "<STR_LIT>", set())<EOL>object.__setattr__(self, "<STR_LIT>", logger)<EOL>


self.output_dim = <NUM_LIT:10><EOL>self.encoder = Chain(self.input_dim).stack(Dense(self.internal_layer_size, '<STR_LIT>'))<EOL>self.decoder = Chain(self.internal_layer_size).stack(Dense(self.input_dim))<EOL>self.classifier = Chain(self.internal_layer_size).stack(Dense(<NUM_LIT:50>, '<STR_LIT>'),<EOL>Dense(self.output_dim),<EOL>Softmax())<EOL>self.register_inner_layers(self.encoder, self.decoder, self.classifier)<EOL>self.target_input = T.ivector('<STR_LIT:target>')<EOL>self.register_external_inputs(self.target_input)<EOL>

All codes that create parameters should be put into 'setup' function.


def _refined_glimpse_sensor(self, x_t, l_p):

<EOL>l_p = l_p * <NUM_LIT> + <NUM_LIT> - <NUM_LIT:4><EOL>l_p = T.cast(T.round(l_p), "<STR_LIT>")<EOL>l_p = l_p * (l_p >= <NUM_LIT:0>)<EOL>l_p = l_p * (l_p < <NUM_LIT>) + (l_p >= <NUM_LIT>) * <NUM_LIT:20><EOL>glimpse_1 = x_t[l_p[<NUM_LIT:0>]: l_p[<NUM_LIT:0>] + <NUM_LIT:7>][:, l_p[<NUM_LIT:1>]: l_p[<NUM_LIT:1>] + <NUM_LIT:7>]<EOL>return glimpse_1<EOL>

Parameters:
    x_t - 28x28 image
    l_p - 2x1 focus vector
Returns:
    7*14 matrix


return T.dot(h_t, self.W_l)<EOL>

Parameters:
    h_t - 256x1 vector
Returns:
    2x1 focus vector


sensor_output = self._refined_glimpse_sensor(x_t, l_p)<EOL>sensor_output = T.flatten(sensor_output)<EOL>h_g = self._relu(T.dot(sensor_output, self.W_g0))<EOL>h_l = self._relu(T.dot(l_p, self.W_g1))<EOL>g = self._relu(T.dot(h_g, self.W_g2_hg) + T.dot(h_l, self.W_g2_hl))<EOL>return g<EOL>

Parameters:
    x_t - 28x28 image
    l_p - 2x1 focus vector
Returns:
    4x12 matrix


@classmethod<EOL><INDENT>def from_mapping(<EOL>cls: Type["<STR_LIT>"], mapping: Optional[Mapping[str, Any]] = None, **kwargs: Any<EOL>) -> "<STR_LIT>":<DEDENT>

mappings: Dict[str, Any] = {}<EOL>if mapping is not None:<EOL><INDENT>mappings.update(mapping)<EOL><DEDENT>mappings.update(kwargs)<EOL>config = cls()<EOL>for key, value in mappings.items():<EOL><INDENT>try:<EOL><INDENT>setattr(config, key, value)<EOL><DEDENT>except AttributeError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return config<EOL>

Create a configuration from a mapping.

        This allows either a mapping to be directly passed or as
        keyword arguments, for example,

        .. code-block:: python

            config = {'keep_alive_timeout': 10}
            Config.from_mapping(config)
            Config.form_mapping(keep_alive_timeout=10)

        Arguments:
            mapping: Optionally a mapping object.
            kwargs: Optionally a collection of keyword arguments to
                form a mapping.


async def serve(<EOL>app: ASGIFramework,<EOL>config: Config,<EOL>*,<EOL>task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED,<EOL>) -> None:

if config.debug:<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>if config.workers != <NUM_LIT:1>:<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>if config.worker_class != "<STR_LIT>":<EOL><INDENT>warnings.warn("<STR_LIT>", Warning)<EOL><DEDENT>await worker_serve(app, config, task_status=task_status)<EOL>

Serve an ASGI framework app given the config.

    This allows for a programmatic way to serve an ASGI framework, it
    can be used via,

    .. code-block:: python

        trio.run(partial(serve, app, config))

    It is assumed that the event-loop is configured before calling
    this function, therefore configuration values that relate to loop
    setup or process setup are ignored.


async def asgi_send(self, message: dict) -> None:

if message["<STR_LIT:type>"] == "<STR_LIT>" and self.state == ASGIHTTPState.REQUEST:<EOL><INDENT>self.response = message<EOL><DEDENT>elif message["<STR_LIT:type>"] == "<STR_LIT>" and self.state in {<EOL>ASGIHTTPState.REQUEST,<EOL>ASGIHTTPState.RESPONSE,<EOL>}:<EOL><INDENT>if self.state == ASGIHTTPState.REQUEST:<EOL><INDENT>headers = build_and_validate_headers(self.response["<STR_LIT>"])<EOL>headers.extend(self.response_headers())<EOL>await self.asend(<EOL>h11.Response(status_code=int(self.response["<STR_LIT:status>"]), headers=headers)<EOL>)<EOL>self.state = ASGIHTTPState.RESPONSE<EOL><DEDENT>if (<EOL>not suppress_body(self.scope["<STR_LIT>"], int(self.response["<STR_LIT:status>"]))<EOL>and message.get("<STR_LIT:body>", b"<STR_LIT>") != b"<STR_LIT>"<EOL>):<EOL><INDENT>await self.asend(h11.Data(data=bytes(message["<STR_LIT:body>"])))<EOL><DEDENT>if not message.get("<STR_LIT>", False):<EOL><INDENT>if self.state != ASGIHTTPState.CLOSED:<EOL><INDENT>await self.asend(h11.EndOfMessage())<EOL>await self.asgi_put({"<STR_LIT:type>": "<STR_LIT>"})<EOL>self.state = ASGIHTTPState.CLOSED<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>raise UnexpectedMessage(self.state, message["<STR_LIT:type>"])<EOL><DEDENT>

Called by the ASGI instance to send a message.


pass<EOL>

Called by the ASGI instance to receive a message.


async def asgi_put(self, message: dict) -> None:

pass<EOL>

Called by the ASGI server to put a message to the ASGI instance.

        See asgi_receive as the get to this put.


@classmethod<EOL><INDENT>def validate_args(cls, tag_name, *args, **kwargs):<DEDENT>

if cls.min_args is not None and len(args) < cls.min_args:<EOL><INDENT>if cls.min_args == <NUM_LIT:1>:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.min_args))<EOL><DEDENT>else:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.min_args))<EOL><DEDENT><DEDENT>if cls.max_args is not None and len(args) > cls.max_args:<EOL><INDENT>if cls.max_args == <NUM_LIT:0>:<EOL><INDENT>if cls.allowed_kwargs:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.allowed_kwargs[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name))<EOL><DEDENT><DEDENT>elif cls.max_args == <NUM_LIT:1>:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.max_args))<EOL><DEDENT>else:<EOL><INDENT>raise TemplateSyntaxError("<STR_LIT>".format(tag_name, cls.max_args))<EOL><DEDENT><DEDENT>

Validate the syntax of the template tag.


if '<STR_LIT>' not in context:<EOL><INDENT>raise ImproperlyConfigured(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>".format(self.tag_name)<EOL>)<EOL><DEDENT>return context['<STR_LIT>']<EOL>

Fetch the request from the context.

This enforces the use of a RequestProcessor, e.g.
::

    render_to_response("page.html", context, context_instance=RequestContext(request))


def get_template_name(self, *tag_args, **tag_kwargs):

return tag_kwargs.get('<STR_LIT>', self.template_name)<EOL>

Get the template name, by default using the :attr:`template_name` attribute.


def render_tag(self, context, *tag_args, **tag_kwargs):

<EOL>if self.as_var:<EOL><INDENT>return BaseAssignmentNode.render_tag(self, context, *tag_args, **tag_kwargs)<EOL><DEDENT>else:<EOL><INDENT>return BaseInclusionNode.render_tag(self, context, *tag_args, **tag_kwargs)<EOL><DEDENT>

Rendering of the tag. It either assigns the value as variable, or renders it.


if self._data is not None:<EOL><INDENT>return self._data["<STR_LIT>"]["<STR_LIT>"]<EOL><DEDENT>


if self._data is not None:<EOL><INDENT>for network in self._data["<STR_LIT>"]:<EOL><INDENT>if network["<STR_LIT>"] == network_id:<EOL><INDENT>return network<EOL><DEDENT><DEDENT><DEDENT>

Function to get specific network (eth0, total, etc)


if self._data is not None:<EOL><INDENT>return_data = int(self._data["<STR_LIT>"]["<STR_LIT>"]) * <NUM_LIT><EOL>if human_readable:<EOL><INDENT>return SynoFormatHelper.bytes_to_readable(<EOL>return_data)<EOL><DEDENT>else:<EOL><INDENT>return return_data<EOL><DEDENT><DEDENT>


var_tb = num / <NUM_LIT> / <NUM_LIT> / <NUM_LIT> / <NUM_LIT><EOL>return round(var_tb, <NUM_LIT:1>)<EOL>


System' percentage of the total cpu load


<EOL>auth = {<EOL>'<STR_LIT>': self.username,<EOL>'<STR_LIT>': self.password,<EOL>}<EOL>return urlencode(auth)<EOL>

Encode user credentials to support special characters.


disk = self._get_disk(disk)<EOL>if disk is not None:<EOL><INDENT>return disk["<STR_LIT:name>"]<EOL><DEDENT>


def _execute_get_url(self, request_url, append_sid=True):

<EOL>self._debuglog("<STR_LIT>" + request_url + "<STR_LIT:'>")<EOL>if append_sid:<EOL><INDENT>self._debuglog("<STR_LIT>" +<EOL>self.access_token + "<STR_LIT>")<EOL>request_url = "<STR_LIT>" % (<EOL>request_url, self.access_token)<EOL><DEDENT>try:<EOL><INDENT>resp = self._session.get(request_url)<EOL>self._debuglog("<STR_LIT>" + str(resp.status_code))<EOL>if resp.status_code == <NUM_LIT:200>:<EOL><INDENT>json_data = json.loads(resp.text)<EOL>if json_data["<STR_LIT:success>"]:<EOL><INDENT>self._debuglog("<STR_LIT>")<EOL>self._debuglog(str(json_data))<EOL>return json_data<EOL><DEDENT>else:<EOL><INDENT>if json_data["<STR_LIT:error>"]["<STR_LIT:code>"] in {<NUM_LIT>, <NUM_LIT>, <NUM_LIT>, <NUM_LIT>}:<EOL><INDENT>self._debuglog("<STR_LIT>" +<EOL>str(json_data["<STR_LIT:error>"]["<STR_LIT:code>"]))<EOL>self._session_error = True<EOL><DEDENT>else:<EOL><INDENT>self._debuglog("<STR_LIT>" + resp.text)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>return None<EOL><DEDENT><DEDENT>except:<EOL><INDENT>return None<EOL><DEDENT>

Function to execute and handle a GET request


if self._utilisation is not None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api,<EOL>self.access_token)<EOL>self._utilisation.update(self._get_url(url))<EOL><DEDENT>if self._storage is not None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api,<EOL>self.access_token)<EOL>self._storage.update(self._get_url(url))<EOL><DEDENT>


if self._data is not None:<EOL><INDENT>disks = []<EOL>for disk in self._data["<STR_LIT>"]:<EOL><INDENT>disks.append(disk["<STR_LIT:id>"])<EOL><DEDENT>return disks<EOL><DEDENT>


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>return volume["<STR_LIT>"]<EOL><DEDENT>

Returns the volume type (RAID1, RAID2, etc)


if num < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>elif num < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>for unit in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>if abs(num) < <NUM_LIT>:<EOL><INDENT>return "<STR_LIT>" % (num, unit)<EOL><DEDENT>num /= <NUM_LIT><EOL><DEDENT>return "<STR_LIT>" % (num, '<STR_LIT>')<EOL>

Converts bytes to a human readable format


def volume_size_total(self, volume, human_readable=True):

volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>return_data = int(volume["<STR_LIT:size>"]["<STR_LIT>"])<EOL>if human_readable:<EOL><INDENT>return SynoFormatHelper.bytes_to_readable(<EOL>return_data)<EOL><DEDENT>else:<EOL><INDENT>return return_data<EOL><DEDENT><DEDENT>


volume = self._get_volume(volume)<EOL>if volume is not None:<EOL><INDENT>vol_disks = volume["<STR_LIT>"]<EOL>if vol_disks is not None:<EOL><INDENT>max_temp = <NUM_LIT:0><EOL>for vol_disk in vol_disks:<EOL><INDENT>disk_temp = self.disk_temp(vol_disk)<EOL>if disk_temp is not None and disk_temp > max_temp:<EOL><INDENT>max_temp = disk_temp<EOL><DEDENT><DEDENT>return max_temp<EOL><DEDENT><DEDENT>

Maximum temperature of all disks making up the volume


if self._storage is None:<EOL><INDENT>api = "<STR_LIT>"<EOL>url = "<STR_LIT>" % (<EOL>self.base_url,<EOL>api)<EOL>self._storage = SynoStorage(self._get_url(url))<EOL><DEDENT>return self._storage<EOL>


disk = self._get_disk(disk)<EOL>if disk is not None:<EOL><INDENT>return disk["<STR_LIT>"]<EOL><DEDENT>


return header_source() + "<STR_LIT:\n>" + classes_source(classes, indent)<EOL>

Convert a set of pyschemas to executable python source code

    Currently supports all built-in types for basic usage.

    Notably not supported:
    * Maintaining class hierarchy
    * Methods, properties and non-field attributes
    * SELF-references


return (<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL>

Get the required header for generated source


d = OrderedDict()<EOL>d["<STR_LIT>"] = repr(self.nullable)<EOL>d["<STR_LIT:default>"] = repr(self.default)<EOL>if self.description is not None:<EOL><INDENT>d["<STR_LIT:description>"] = repr(self.description)<EOL><DEDENT>return d<EOL>

Return a dictionary the field definition

        Should contain all fields that are required for the definition of this field in a pyschema class


d = {}<EOL>for fname, f in record._fields.iteritems():<EOL><INDENT>val = getattr(record, fname)<EOL>if val is not None:<EOL><INDENT>d[fname] = f.dump(val)<EOL><DEDENT><DEDENT>return d<EOL>

Dump record in json-encodable object format


kwargs = {}<EOL>for key in dct:<EOL><INDENT>field_type = schema._fields.get(key)<EOL>if field_type is None:<EOL><INDENT>raise ParseError("<STR_LIT>" % (schema.__name__, key))<EOL><DEDENT>kwargs[key] = field_type.load(dct[key])<EOL><DEDENT>return schema(**kwargs)<EOL>


schema = jsonschema.get_root_schema_dict(valid_record)<EOL>record_json = jsonschema.dumps(valid_record)<EOL>record_dict = json.loads(record_json)<EOL>validate(record_dict, schema)<EOL>

Serialize and validate a record

        The record is dumped to JSON and then loaded into a Python dictionary,
        which is checked against the known schema


state = SchemaGeneratorState()<EOL>schema = get_schema_dict(record, state)<EOL>del state.record_schemas[record._schema_name]<EOL>if state.record_schemas:<EOL><INDENT>schema['<STR_LIT>'] = dict()<EOL>for name, sub_schema in state.record_schemas.iteritems():<EOL><INDENT>schema['<STR_LIT>'][name] = sub_schema<EOL><DEDENT><DEDENT>return schema<EOL>

Return a root jsonschema for a given record

    A root schema includes the $schema attribute and all sub-record
    schemas and definitions.


def mr_writer(job, outputs, output_stream,<EOL>stderr=sys.stderr, dumps=core.dumps):

for output in outputs:<EOL><INDENT>try:<EOL><INDENT>print(dumps(output), file=output_stream)<EOL><DEDENT>except core.ParseError as e:<EOL><INDENT>print(e, file=stderr)<EOL>raise<EOL><DEDENT><DEDENT>

Writes a stream of json serialised pyschema Records to a file object

    Can be used as job.writer in luigi.hadoop.JobTask


if isinstance(schema_string, str):<EOL><INDENT>schema_string = schema_string.decode("<STR_LIT:utf8>")<EOL><DEDENT>schema_struct = json.loads(schema_string)<EOL>return AvroSchemaParser().parse_schema_struct(schema_struct)<EOL>

Load and return a PySchema class from an avsc string


def select_renderer(request: web.Request, renderers: OrderedDict, force=True):

header = request.headers.get('<STR_LIT>', '<STR_LIT>')<EOL>best_match = mimeparse.best_match(renderers.keys(), header)<EOL>if not best_match or best_match not in renderers:<EOL><INDENT>if force:<EOL><INDENT>return tuple(renderers.items())[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>raise web.HTTPNotAcceptable<EOL><DEDENT><DEDENT>return best_match, renderers[best_match]<EOL>

Given a request, a list of renderers, and the ``force`` configuration
option, return a two-tuple of:
(media type, render callable). Uses mimeparse to find the best media
type match from the ACCEPT header.


@contextmanager<EOL>def add_resource_context(<EOL>app: web.Application, module=None,<EOL>url_prefix: str=None, name_prefix: str=None, make_resource=lambda cls: cls()<EOL>):

assert isinstance(app.router, ResourceRouter), '<STR_LIT>'<EOL>if isinstance(module, (str, bytes)):<EOL><INDENT>module = importlib.import_module(module)<EOL><DEDENT>def get_base_name(resource, method_name, names):<EOL><INDENT>return names.get(method_name,<EOL>app.router.get_default_handler_name(resource, method_name))<EOL><DEDENT>default_make_resource = make_resource<EOL>def add_route(<EOL>path: str, resource, names: Mapping=None, make_resource=None<EOL>):<EOL><INDENT>make_resource = make_resource or default_make_resource<EOL>names = names or {}<EOL>if isinstance(resource, (str, bytes)):<EOL><INDENT>if not module:<EOL><INDENT>raise ValueError(<EOL>'<STR_LIT>'<EOL>)<EOL><DEDENT>resource_cls = getattr(module, resource)<EOL>resource = make_resource(resource_cls)<EOL><DEDENT>path = make_path(path, url_prefix)<EOL>if name_prefix:<EOL><INDENT>supported_method_names = get_supported_method_names(resource)<EOL>names = {<EOL>method_name: '<STR_LIT:.>'.join(<EOL>(name_prefix, get_base_name(resource, method_name, names=names))<EOL>)<EOL>for method_name in supported_method_names<EOL>}<EOL><DEDENT>return app.router.add_resource_object(path, resource, names=names)<EOL><DEDENT>yield add_route<EOL>

Context manager which yields a function for adding multiple resources from a given module
    to an app using `ResourceRouter <aiohttp_utils.routing.ResourceRouter>`.

    Example:

    .. code-block:: python

        # myapp/articles/views.py
        class ArticleList:
            async def get(self, request):
                return web.Response(b'article list...')

        class ArticleDetail:
            async def get(self, request):
                return web.Response(b'article detail...')

    .. code-block:: python

        # myapp/app.py
        from myapp.articles import views

        with add_resource_context(app, url_prefix='/api/') as route:
            route('/articles/', views.ArticleList())
            route('/articles/{pk}', views.ArticleDetail())

        app.router['ArticleList:get'].url()  # /api/articles/
        app.router['ArticleDetail:get'].url(parts={'pk': 42})  # /api/articles/42

    If you prefer, you can also pass module and class names as strings. ::

        with add_resource_context(app, module='myapp.articles.views',
                                  url_prefix='/api/') as route:
            route('/articles/', 'ArticleList')
            route('/articles/{pk}', 'ArticleDetail')

    .. note::
        If passing class names, the resource classes will be instantiated with no
        arguments. You can change this behavior by overriding ``make_resource``.

        .. code-block:: python

            # myapp/authors/views.py
            class AuthorList:
                def __init__(self, db):
                    self.db = db

                async def get(self, request):
                    # Fetch authors from self.db...

        .. code-block:: python

            # myapp/app.py
            from myapp.database import db

            with add_resource_context(app, module='myapp.authors.views',
                                    url_prefix='/api/',
                                    make_resource=lambda cls: cls(db=db)) as route:
                route('/authors/', 'AuthorList')

    :param app: Application to add routes to.
    :param resource: Import path to module (str) or module object
        which contains the resource classes.
    :param url_prefix: Prefix to prepend to all route paths.
    :param name_prefix: Prefix to prepend to all route names.
    :param make_resource: Function which receives a resource class and returns
        a resource instance.


return SystemImportNode.handle_token(parser, token)<EOL>

Import a Javascript module via SystemJS, bundling the app.

Syntax::

    {% systemjs_import 'path/to/file' %}

Example::

    {% systemjs_import 'mydjangoapp/js/myapp' %}

Which would be rendered like::

    <script type="text/javascript" src="/static/CACHE/mydjangoapp.js.min.myapp.js"></script>

where /static/CACHE can be configured through settings.

In DEBUG mode, the result would be

    <script type="text/javascript">System.import('mydjangoapp/js/myapp.js');</script>


module_path = self.path.resolve(context)<EOL>if not settings.SYSTEMJS_ENABLED:<EOL><INDENT>if settings.SYSTEMJS_DEFAULT_JS_EXTENSIONS:<EOL><INDENT>name, ext = posixpath.splitext(module_path)<EOL>if not ext:<EOL><INDENT>module_path = '<STR_LIT>'.format(module_path)<EOL><DEDENT><DEDENT>if settings.SYSTEMJS_SERVER_URL:<EOL><INDENT>tpl = """<STR_LIT>"""<EOL><DEDENT>else:<EOL><INDENT>tpl = """<STR_LIT>"""<EOL><DEDENT>return tpl.format(app=module_path, url=settings.SYSTEMJS_SERVER_URL)<EOL><DEDENT>rel_path = System.get_bundle_path(module_path)<EOL>url = staticfiles_storage.url(rel_path)<EOL>tag_attrs = {'<STR_LIT:type>': '<STR_LIT>'}<EOL>for key, value in self.tag_attrs.items():<EOL><INDENT>if not isinstance(value, bool):<EOL><INDENT>value = value.resolve(context)<EOL><DEDENT>tag_attrs[key] = value<EOL><DEDENT>return """<STR_LIT>""".format(<EOL>url=url, attrs=flatatt(tag_attrs)<EOL>)<EOL>

Build the filepath by appending the extension.


<EOL>_manifest_name = self.manifest_name<EOL>self.manifest_name = self.systemjs_manifest_name<EOL>bundle_files = self.load_manifest()<EOL>self.manifest_name = _manifest_name<EOL>for file, hashed_file in bundle_files.copy().items():<EOL><INDENT>if not self.exists(file) or not self.exists(hashed_file):<EOL><INDENT>del bundle_files[file]<EOL><DEDENT><DEDENT>return bundle_files<EOL>

Load the existing systemjs manifest and remove any entries that no longer
exist on the storage.


def __init__(self, system, app, **options):

self.system = system<EOL>self.app = app<EOL>options.setdefault('<STR_LIT>', settings.SYSTEMJS_JSPM_EXECUTABLE)<EOL>self.opts = options<EOL>bundle_cmd = self.get_bundle_sfx_cmd() if self.opts.get('<STR_LIT>') else '<STR_LIT>'<EOL>self.command = '<STR_LIT>' + bundle_cmd + '<STR_LIT>'<EOL>self.stdout = self.stdin = self.stderr = subprocess.PIPE<EOL>

Initialize a SystemBundle object.

:param system: a System instance that holds the non-bundle specific
meta information (such as jspm version, configuration)

:param app: string, the name of the JS package to bundle. This may be
missing the '.js' extension.

:param options: dict containing the bundle-specific options. Possible
options:
    `jspm`: `jspm` executable (if it's not on $PATH, for example)
    `log`: logging mode for jspm, can be ok|warn|err. Only available
           for jspm >= 0.16.3
    `minify`: boolean, whether go generate minified bundles or not
    `sfx`: boolean, generate a self-executing bundle or not
    `skip-source-maps: boolean, whether to generate source maps or not


repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))<EOL>git_log = subprocess.Popen(<EOL>'<STR_LIT>',<EOL>stdout=subprocess.PIPE, stderr=subprocess.PIPE,<EOL>shell=True, cwd=repo_dir, universal_newlines=True<EOL>)<EOL>timestamp = git_log.communicate()[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>timestamp = datetime.datetime.utcfromtimestamp(int(timestamp))<EOL><DEDENT>except ValueError:<EOL><INDENT>return None<EOL><DEDENT>return timestamp.strftime('<STR_LIT>')<EOL>

Returns a numeric identifier of the latest git changeset.

    The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format.
    This value isn't guaranteed to be unique, but collisions are very unlikely,
    so it's sufficient for generating the development version numbers.


for k, v in merge_dct.items():<EOL><INDENT>if (k in self and isinstance(self[k], dict) and isinstance(merge_dct[k], collections.Mapping)):<EOL><INDENT>self[k].merge_(dicto(merge_dct[k]))<EOL><DEDENT>else:<EOL><INDENT>self[k] = merge_dct[k]<EOL><DEDENT><DEDENT>return self<EOL>

Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
        updating only top-level keys, dict_merge recurses down into dicts nested
        to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
        ``self``.
        :param self: dict onto which the merge is executed
        :param merge_dct: self merged into self
        :return: None


if not isinstance(dicto, Dicto):<EOL><INDENT>dicto = Dicto(dicto)<EOL><DEDENT>if not isinstance(other, Dicto):<EOL><INDENT>other = Dicto(other)<EOL><DEDENT>for k, v in other.__dict__.items():<EOL><INDENT>if k in dicto and isinstance(dicto[k], Dicto) and isinstance(other[k], Dicto):<EOL><INDENT>dicto[k] = merge(dicto[k], other[k])<EOL><DEDENT>else:<EOL><INDENT>dicto[k] = other[k]<EOL><DEDENT><DEDENT>return dicto<EOL>

Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
    updating only top-level keys, dict_merge recurses down into dicts nested
    to an arbitrary depth, updating keys. The ``other`` is merged into
    ``dicto``.
    :param dicto: dict onto which the merge is executed
    :param other: dict that is going to merged into dicto
    :return: None


def visualize(spect, frequencies, title="<STR_LIT>"):

i = <NUM_LIT:0><EOL>for freq, (index, row) in zip(frequencies[::-<NUM_LIT:1>], enumerate(spect[::-<NUM_LIT:1>, :])):<EOL><INDENT>plt.subplot(spect.shape[<NUM_LIT:0>], <NUM_LIT:1>, index + <NUM_LIT:1>)<EOL>if i == <NUM_LIT:0>:<EOL><INDENT>plt.title(title)<EOL>i += <NUM_LIT:1><EOL><DEDENT>plt.ylabel("<STR_LIT>".format(freq))<EOL>plt.plot(row)<EOL><DEDENT>plt.show()<EOL>

Visualize the result of calling seg.filter_bank() for any number of filters


def generate_frames_as_segments(self, frame_duration_ms, zero_pad=True):

for frame in self.generate_frames(frame_duration_ms, zero_pad=zero_pad):<EOL><INDENT>seg = AudioSegment(pydub.AudioSegment(data=frame.bytes, sample_width=self.sample_width,<EOL>frame_rate=self.frame_rate, channels=self.channels), self.name)<EOL>yield seg, frame.timestamp<EOL><DEDENT>

Does the same thing as `generate_frames`, but yields tuples of (AudioSegment, timestamp) instead of Frames.


def _execute_sox_cmd(self, cmd, console_output=False):

on_windows = platform.system().lower() == "<STR_LIT>"<EOL>def _get_random_tmp_file():<EOL><INDENT>if on_windows:<EOL><INDENT>rand_string = "<STR_LIT>".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(<NUM_LIT:8>))<EOL>tmp = self.name + "<STR_LIT:_>" + rand_string<EOL>WinTempFile = collections.namedtuple("<STR_LIT>", "<STR_LIT:name>")<EOL>tmp = WinTempFile(tmp)<EOL><DEDENT>else:<EOL><INDENT>tmp = tempfile.NamedTemporaryFile()<EOL><DEDENT>return tmp<EOL><DEDENT>tmp = _get_random_tmp_file()<EOL>othertmp = _get_random_tmp_file()<EOL>self.export(tmp.name, format="<STR_LIT>")<EOL>stdout = stderr = subprocess.PIPE if console_output else subprocess.DEVNULL<EOL>command = cmd.format(inputfile=tmp.name, outputfile=othertmp.name)<EOL>res = subprocess.call(command.split('<STR_LIT:U+0020>'), stdout=stdout, stderr=stderr)<EOL>assert res == <NUM_LIT:0>, "<STR_LIT>"<EOL>other = AudioSegment(pydub.AudioSegment.from_wav(othertmp.name), self.name)<EOL>if on_windows:<EOL><INDENT>os.remove(tmp.name)<EOL>os.remove(othertmp.name)<EOL><DEDENT>else:<EOL><INDENT>tmp.close()<EOL>othertmp.close()<EOL><DEDENT>return other<EOL>

Executes a Sox command in a platform-independent manner.

`cmd` must be a format string that includes {inputfile} and {outputfile}.


def filter_bank(self, lower_bound_hz=<NUM_LIT:50>, upper_bound_hz=<NUM_LIT>, nfilters=<NUM_LIT>, mode='<STR_LIT>'):

<EOL>data = self.to_numpy_array()<EOL>if mode.lower() == '<STR_LIT>':<EOL><INDENT>frequencies = librosa.core.mel_frequencies(n_mels=nfilters, fmin=lower_bound_hz, fmax=upper_bound_hz)<EOL><DEDENT>elif mode.lower() == '<STR_LIT>':<EOL><INDENT>frequencies = np.linspace(lower_bound_hz, upper_bound_hz, num=nfilters, endpoint=True)<EOL><DEDENT>elif mode.lower() == '<STR_LIT>':<EOL><INDENT>start = np.log10(lower_bound_hz)<EOL>stop = np.log10(upper_bound_hz)<EOL>frequencies = np.logspace(start, stop, num=nfilters, endpoint=True, base=<NUM_LIT>)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>".format(mode))<EOL><DEDENT>rows = [filters.bandpass_filter(data, freq*<NUM_LIT>, freq*<NUM_LIT>, self.frame_rate) for freq in frequencies]<EOL>rows = np.array(rows)<EOL>spect = np.vstack(rows)<EOL>return spect, frequencies<EOL>

Returns a numpy array of shape (nfilters, nsamples), where each
row of data is the result of bandpass filtering the audiosegment
around a particular frequency. The frequencies are
spaced from `lower_bound_hz` to `upper_bound_hz` and are returned with
the np array. The particular spacing of the frequencies depends on `mode`,
which can be either: 'linear', 'mel', or 'log'.

.. note:: This method is an approximation of a gammatone filterbank
          until I get around to writing an actual gammatone filterbank
          function.

.. code-block:: python

    # Example usage
    import audiosegment
    import matplotlib.pyplot as plt
    import numpy as np

    def visualize(spect, frequencies, title=""):
        # Visualize the result of calling seg.filter_bank() for any number of filters
        i = 0
        for freq, (index, row) in zip(frequencies[::-1], enumerate(spect[::-1, :])):
            plt.subplot(spect.shape[0], 1, index + 1)
            if i == 0:
                plt.title(title)
                i += 1
            plt.ylabel("{0:.0f}".format(freq))
            plt.plot(row)
        plt.show()

    seg = audiosegment.from_file("some_audio.wav").resample(sample_rate_Hz=24000, sample_width=2, channels=1)
    spec, frequencies = seg.filter_bank(nfilters=5)
    visualize(spec, frequencies)

.. image:: images/filter_bank.png

:param lower_bound_hz:  The lower bound of the frequencies to use in the bandpass filters.
:param upper_bound_hz:  The upper bound of the frequencies to use in the bandpass filters.
:param nfilters:        The number of filters to apply. This will determine which frequencies
                        are used as well, as they are interpolated between
                        `lower_bound_hz` and `upper_bound_hz` based on `mode`.
:param mode:            The way the frequencies are spaced. Options are: `linear`, in which case
                        the frequencies are linearly interpolated between `lower_bound_hz` and
                        `upper_bound_hz`, `mel`, in which case the mel frequencies are used,
                        or `log`, in which case they are log-10 spaced.
:returns:               A numpy array of the form (nfilters, nsamples), where each row is the
                        audiosegment, bandpass-filtered around a particular frequency,
                        and the list of frequencies. I.e., returns (spec, freqs).


return {'<STR_LIT:name>': self.name, '<STR_LIT>': self.seg}<EOL>

Serializes into a dict for the pickle protocol.

:returns: The dict to pickle.


ret = AudioSegment(self.seg, self.name)<EOL>selfdata = [self.seg._data]<EOL>otherdata = [o.seg._data for o in others]<EOL>ret.seg._data = b'<STR_LIT>'.join(selfdata + otherdata)<EOL>return ret<EOL>

Reduces others into this one by concatenating all the others onto this one and
returning the result. Does not modify self, instead, makes a copy and returns that.

:param others: The other AudioSegment objects to append to this one.
:returns: The concatenated result.


Deserializes from a dict for the pickle protocol.

:param d: The dict to unpickle from.


def generate_frames(self, frame_duration_ms, zero_pad=True):

Frame = collections.namedtuple("<STR_LIT>", "<STR_LIT>")<EOL>bytes_per_frame = int(self.frame_rate * (frame_duration_ms / <NUM_LIT:1000>) * self.sample_width)<EOL>offset = <NUM_LIT:0>  <EOL>timestamp = <NUM_LIT:0.0>  <EOL>frame_duration_s = (bytes_per_frame / self.frame_rate) / self.sample_width<EOL>while offset + bytes_per_frame < len(self.raw_data):<EOL><INDENT>yield Frame(self.raw_data[offset:offset + bytes_per_frame], timestamp, frame_duration_s)<EOL>timestamp += frame_duration_s<EOL>offset += bytes_per_frame<EOL><DEDENT>if zero_pad:<EOL><INDENT>rest = self.raw_data[offset:]<EOL>zeros = bytes(bytes_per_frame - len(rest))<EOL>yield Frame(rest + zeros, timestamp, frame_duration_s)<EOL><DEDENT>

Yields self's data in chunks of frame_duration_ms.

This function adapted from pywebrtc's example [https://github.com/wiseman/py-webrtcvad/blob/master/example.py].

:param frame_duration_ms: The length of each frame in ms.
:param zero_pad: Whether or not to zero pad the end of the AudioSegment object to get all
                 the audio data out as frames. If not, there may be a part at the end
                 of the Segment that is cut off (the part will be <= `frame_duration_ms` in length).
:returns: A Frame object with properties 'bytes (the data)', 'timestamp (start time)', and 'duration'.


def spectrogram(self, start_s=None, duration_s=None, start_sample=None, num_samples=None,<EOL>window_length_s=None, window_length_samples=None, overlap=<NUM_LIT:0.5>, window=('<STR_LIT>', <NUM_LIT>)):

if start_s is not None and start_sample is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if duration_s is not None and num_samples is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if window_length_s is not None and window_length_samples is not None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if window_length_s is None and window_length_samples is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>if start_s is None and start_sample is None:<EOL><INDENT>start_sample = <NUM_LIT:0><EOL><DEDENT>elif start_s is not None:<EOL><INDENT>start_sample = int(round(start_s * self.frame_rate))<EOL><DEDENT>if duration_s is None and num_samples is None:<EOL><INDENT>num_samples = len(self.get_array_of_samples()) - int(start_sample)<EOL><DEDENT>elif duration_s is not None:<EOL><INDENT>num_samples = int(round(duration_s * self.frame_rate))<EOL><DEDENT>if window_length_s is not None:<EOL><INDENT>window_length_samples = int(round(window_length_s * self.frame_rate))<EOL><DEDENT>if start_sample + num_samples > len(self.get_array_of_samples()):<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>arr = self.to_numpy_array()[start_sample:start_sample+num_samples]<EOL>fs, ts, sxx = signal.spectrogram(arr, self.frame_rate, scaling='<STR_LIT>', nperseg=window_length_samples,<EOL>noverlap=int(round(overlap * window_length_samples)),<EOL>mode='<STR_LIT>', window=window)<EOL>return fs, ts, sxx<EOL>

Does a series of FFTs from `start_s` or `start_sample` for `duration_s` or `num_samples`.
Effectively, transforms a slice of the AudioSegment into the frequency domain across different
time bins.

.. code-block:: python

    # Example for plotting a spectrogram using this function
    import audiosegment
    import matplotlib.pyplot as plt

    #...
    seg = audiosegment.from_file("somebodytalking.wav")
    freqs, times, amplitudes = seg.spectrogram(window_length_s=0.03, overlap=0.5)
    amplitudes = 10 * np.log10(amplitudes + 1e-9)

    # Plot
    plt.pcolormesh(times, freqs, amplitudes)
    plt.xlabel("Time in Seconds")
    plt.ylabel("Frequency in Hz")
    plt.show()

.. image:: images/spectrogram.png

:param start_s: The start time. Starts at the beginning if neither this nor `start_sample` is specified.
:param duration_s: The duration of the spectrogram in seconds. Goes to the end if neither this nor
                   `num_samples` is specified.
:param start_sample: The index of the first sample to use. Starts at the beginning if neither this nor
                     `start_s` is specified.
:param num_samples: The number of samples in the spectrogram. Goes to the end if neither this nor
                    `duration_s` is specified.
:param window_length_s: The length of each FFT in seconds. If the total number of samples in the spectrogram
                        is not a multiple of the window length in samples, the last window will be zero-padded.
:param window_length_samples: The length of each FFT in number of samples. If the total number of samples in the
                        spectrogram is not a multiple of the window length in samples, the last window will
                        be zero-padded.
:param overlap: The fraction of each window to overlap.
:param window: See Scipy's spectrogram-function_.
               This parameter is passed as-is directly into the Scipy spectrogram function. It's documentation is reproduced here:
               Desired window to use. If window is a string or tuple, it is passed to get_window to generate the window values,
               which are DFT-even by default. See get_window for a list of windows and required parameters.
               If window is array_like it will be used directly as the window and its length must be
               `window_length_samples`.
               Defaults to a Tukey window with shape parameter of 0.25.
:returns: Three np.ndarrays: The frequency values in Hz (the y-axis in a spectrogram), the time values starting
          at start time and then increasing by `duration_s` each step (the x-axis in a spectrogram), and
          the dB of each time/frequency bin as a 2D array of shape [len(frequency values), len(duration)].
:raises ValueError: If `start_s` and `start_sample` are both specified, if `duration_s` and `num_samples` are both
                    specified, if the first window's duration plus start time lead to running off the end
                    of the AudioSegment, or if `window_length_s` and `window_length_samples` are either
                    both specified or if they are both not specified.

.. _spectrogram-function: https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.spectrogram.html


def _update_segmentation_mask(segmentation_mask, onset_fronts, offset_fronts, onset_front_id, offset_front_id_most_overlap):

<EOL>onset_front_overlap, offset_front_overlap = _get_consecutive_and_overlapping_fronts(onset_fronts, offset_fronts, onset_front_id, offset_front_id_most_overlap)<EOL>onset_front = _get_front_idxs_from_id(onset_fronts, onset_front_id)<EOL>offset_front = _get_front_idxs_from_id(offset_fronts, offset_front_id_most_overlap)<EOL>msg = "<STR_LIT>".format(<EOL>onset_front, offset_front, onset_front_overlap, offset_front_overlap<EOL>)<EOL>assert onset_front_overlap, msg<EOL>assert offset_front_overlap, msg<EOL>onset_front = onset_front_overlap<EOL>offset_front = offset_front_overlap<EOL>flow_on, _slow_on = onset_front[<NUM_LIT:0>]<EOL>fhigh_on, _shigh_on = onset_front[-<NUM_LIT:1>]<EOL>flow_off, _slow_off = offset_front[<NUM_LIT:0>]<EOL>fhigh_off, _shigh_off = offset_front[-<NUM_LIT:1>]<EOL>flow = max(flow_on, flow_off)<EOL>fhigh = min(fhigh_on, fhigh_off)<EOL>for fidx, _freqchan in enumerate(segmentation_mask[flow:fhigh + <NUM_LIT:1>, :], start=flow):<EOL><INDENT>assert fidx >= flow, "<STR_LIT>".format(fidx, flow)<EOL>assert (fidx - flow) < len(onset_front), "<STR_LIT>".format(<EOL>fidx, flow, len(onset_front), onset_front<EOL>)<EOL>assert (fidx - flow) < len(offset_front), "<STR_LIT>".format(<EOL>fidx, flow, len(offset_front), offset_front<EOL>)<EOL>_, beg = onset_front[fidx - flow]<EOL>_, end = offset_front[fidx - flow]<EOL>if beg > end:<EOL><INDENT>end, beg = beg, end<EOL><DEDENT>assert end >= beg<EOL>segmentation_mask[fidx, beg:end + <NUM_LIT:1>] = onset_front_id<EOL>onset_fronts[fidx, (beg + <NUM_LIT:1>):(end + <NUM_LIT:1>)] = <NUM_LIT:0><EOL>offset_fronts[fidx, (beg + <NUM_LIT:1>):(end + <NUM_LIT:1>)] = <NUM_LIT:0><EOL><DEDENT>nfreqs_used_in_onset_front = (fidx - flow) + <NUM_LIT:1><EOL>indexes = np.arange(flow, fhigh + <NUM_LIT:1>, <NUM_LIT:1>, dtype=np.int64)<EOL>onset_front_sample_idxs_across_freqs = np.array([s for _, s in onset_front])<EOL>onset_front_sample_idxs_across_freqs_up_to_break = onset_front_sample_idxs_across_freqs[:nfreqs_used_in_onset_front]<EOL>offset_front_sample_idxs_across_freqs = np.array([s for _, s in offset_front])<EOL>offset_front_sample_idxs_across_freqs_up_to_break = offset_front_sample_idxs_across_freqs[:nfreqs_used_in_onset_front]<EOL>offset_fronts[indexes[:nfreqs_used_in_onset_front], offset_front_sample_idxs_across_freqs_up_to_break] = <NUM_LIT:0><EOL>onset_fronts[indexes[:nfreqs_used_in_onset_front], onset_front_sample_idxs_across_freqs_up_to_break] = <NUM_LIT:0><EOL>whole_onset_front_matched = onset_front_id not in np.unique(onset_fronts)<EOL>return whole_onset_front_matched<EOL>

Returns an updated segmentation mask such that the input `segmentation_mask` has been updated by segmenting between
`onset_front_id` and `offset_front_id`, as found in `onset_fronts` and `offset_fronts`, respectively.

This function also returns the onset_fronts and offset_fronts matrices, updated so that any fronts that are of
less than 3 channels wide are removed.

This function also returns a boolean value indicating whether the onset channel went to completion.

Specifically, segments by doing the following:

- Going across frequencies in the onset_front,
- add the segment mask ID (the onset front ID) to all samples between the onset_front and the offset_front,
  if the offset_front is in that frequency.

Possible scenarios:

Fronts line up completely:

::

    |   |       S S S
    |   |  =>   S S S
    |   |       S S S
    |   |       S S S

Onset front starts before offset front:

::

    |           |
    |   |       S S S
    |   |  =>   S S S
    |   |       S S S

Onset front ends after offset front:

::

    |   |       S S S
    |   |  =>   S S S
    |   |       S S S
    |           |

Onset front starts before and ends after offset front:

::

    |           |
    |   |  =>   S S S
    |   |       S S S
    |           |

The above three options in reverse:

::

        |       |S S|           |
    |S S|       |S S|       |S S|
    |S S|       |S S|       |S S|
    |S S|           |           |

There is one last scenario:

::

    |   |
    \   /
     \ /
     / \
    |   |

Where the offset and onset fronts cross one another. If this happens, we simply
reverse the indices and accept:

::

    |sss|
    \sss/
     \s/
     /s\
    |sss|

The other option would be to destroy the offset front from the crossover point on, and
then search for a new offset front for the rest of the onset front.


def _form_onset_offset_fronts(ons_or_offs, sample_rate_hz, threshold_ms=<NUM_LIT:20>):

threshold_s = threshold_ms / <NUM_LIT:1000><EOL>threshold_samples = sample_rate_hz * threshold_s<EOL>ons_or_offs = np.copy(ons_or_offs)<EOL>claimed = []<EOL>this_id = <NUM_LIT:2><EOL>for frequency_index, row in enumerate(ons_or_offs[:, :]):<EOL><INDENT>ones = np.reshape(np.where(row == <NUM_LIT:1>), (-<NUM_LIT:1>,))<EOL>for top_level_frequency_one_index in ones:<EOL><INDENT>claimed.append((frequency_index, top_level_frequency_one_index))<EOL>found_a_front = False<EOL>for other_frequency_index, other_row in enumerate(ons_or_offs[frequency_index + <NUM_LIT:1>:, :], start=frequency_index + <NUM_LIT:1>):<EOL><INDENT>upper_limit_index = top_level_frequency_one_index + threshold_samples<EOL>lower_limit_index = top_level_frequency_one_index - threshold_samples<EOL>other_ones = np.reshape(np.where(other_row == <NUM_LIT:1>), (-<NUM_LIT:1>,))  <EOL>tmp = np.reshape(np.where((other_ones >= lower_limit_index)  <EOL>& (other_ones <= upper_limit_index)), (-<NUM_LIT:1>,))<EOL>other_ones = other_ones[tmp]  <EOL>if len(other_ones) > <NUM_LIT:0>:<EOL><INDENT>unclaimed_idx = other_ones[<NUM_LIT:0>]  <EOL>claimed.append((other_frequency_index, unclaimed_idx))<EOL><DEDENT>elif len(claimed) < <NUM_LIT:3>:<EOL><INDENT>ons_or_offs[frequency_index, top_level_frequency_one_index] = <NUM_LIT:0><EOL>claimed = []<EOL>break  <EOL><DEDENT>elif len(claimed) >= <NUM_LIT:3>:<EOL><INDENT>found_a_front = True<EOL>claimed_as_indexes = tuple(np.array(claimed).T)<EOL>ons_or_offs[claimed_as_indexes] = this_id<EOL>this_id += <NUM_LIT:1><EOL>claimed = []<EOL>break  <EOL><DEDENT><DEDENT>if len(claimed) >= <NUM_LIT:3>:<EOL><INDENT>claimed_as_indexes = tuple(np.array(claimed).T)<EOL>ons_or_offs[claimed_as_indexes] = this_id<EOL>this_id += <NUM_LIT:1><EOL>claimed = []<EOL><DEDENT>elif found_a_front:<EOL><INDENT>this_id += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>ons_or_offs[frequency_index, top_level_frequency_one_index] = <NUM_LIT:0><EOL>claimed = []<EOL><DEDENT><DEDENT><DEDENT>return ons_or_offs<EOL>

Takes an array of onsets or offsets (shape = [nfrequencies, nsamples], where a 1 corresponds to an on/offset,
and samples are 0 otherwise), and returns a new array of the same shape, where each 1 has been replaced by
either a 0, if the on/offset has been discarded, or a non-zero positive integer, such that
each front within the array has a unique ID - for example, all 2s in the array will be the front for on/offset
front 2, and all the 15s will be the front for on/offset front 15, etc.

Due to implementation details, there will be no 1 IDs.


fidx, sidx = index<EOL>id = front[fidx, sidx]<EOL>if id == <NUM_LIT:0>:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>return id<EOL><DEDENT>

Returns the front ID found in `front` at the given `index`.

:param front:               An onset or offset front array of shape [nfrequencies, nsamples]
:index:                     A tuple of the form (frequency index, sample index)
:returns:                   The ID of the front or -1 if not found in `front` and the item at `onsets_or_offsets[index]`
                            is not a 1.


def _downsample_one_or_the_other(mask, mask_indexes, stft, stft_indexes):

assert len(mask.shape) == <NUM_LIT:2>, "<STR_LIT>".format(len(mask.shape))<EOL>assert len(stft.shape) == <NUM_LIT:2>, "<STR_LIT>".format(len(stft.shape))<EOL>if mask.shape[<NUM_LIT:1>] > stft.shape[<NUM_LIT:1>]:<EOL><INDENT>downsample_factor = mask.shape[<NUM_LIT:1>] / stft.shape[<NUM_LIT:1>]<EOL>indexes = _get_downsampled_indexes(mask, downsample_factor)<EOL>mask = mask[:, indexes]<EOL>mask_indexes = np.array(indexes)<EOL><DEDENT>elif mask.shape[<NUM_LIT:1>] < stft.shape[<NUM_LIT:1>]:<EOL><INDENT>downsample_factor = stft.shape[<NUM_LIT:1>] / mask.shape[<NUM_LIT:1>]<EOL>indexes = _get_downsampled_indexes(stft, downsample_factor)<EOL>stft = stft[:, indexes]<EOL>stft_indexes = np.array(indexes)<EOL><DEDENT>return mask, mask_indexes, stft, stft_indexes<EOL>

Takes the given `mask` and `stft`, which must be matrices of shape `frequencies, times`
and downsamples one of them into the other one's times, so that the time dimensions
are equal. Leaves the frequency dimension untouched.


def _choose_front_id_from_candidates(candidate_offset_front_ids, offset_fronts, offsets_corresponding_to_onsets):

noverlaps = []  <EOL>for offset_front_id in candidate_offset_front_ids:<EOL><INDENT>offset_front_f_idxs, offset_front_s_idxs = np.where(offset_fronts == offset_front_id)<EOL>offset_front_idxs = [(f, i) for f, i in zip(offset_front_f_idxs, offset_front_s_idxs)]<EOL>noverlap_this_id = len(set(offset_front_idxs).symmetric_difference(set(offsets_corresponding_to_onsets)))<EOL>noverlaps.append((noverlap_this_id, offset_front_id))<EOL><DEDENT>_overlapped, chosen_offset_front_id = max(noverlaps, key=lambda t: t[<NUM_LIT:0>])<EOL>return int(chosen_offset_front_id)<EOL>

Returns a front ID which is the id of the offset front that contains the most overlap
with offsets that correspond to the given onset front ID.


<EOL>gradient = np.nan_to_num(np.apply_along_axis(np.gradient, <NUM_LIT:1>, s), copy=False)<EOL>threshold = np.squeeze(np.nanmean(gradient, axis=<NUM_LIT:1>) + np.nanstd(gradient, axis=<NUM_LIT:1>))<EOL>half_window = <NUM_LIT:4><EOL>if do_peaks:<EOL><INDENT>indexes = [signal.argrelextrema(gradient[i, :], np.greater, order=half_window)[<NUM_LIT:0>] for i in range(gradient.shape[<NUM_LIT:0>])]<EOL><DEDENT>else:<EOL><INDENT>indexes = [signal.argrelextrema(gradient[i, :], np.less, order=half_window)[<NUM_LIT:0>] for i in range(gradient.shape[<NUM_LIT:0>])]<EOL><DEDENT>extrema = np.zeros(s.shape)<EOL>for row_index, index_array in enumerate(indexes):<EOL><INDENT>for col_index in index_array:<EOL><INDENT>if do_peaks and (gradient[row_index, col_index] > threshold[row_index]):<EOL><INDENT>extrema[row_index, col_index] = <NUM_LIT:1><EOL><DEDENT>elif not do_peaks:<EOL><INDENT>extrema[row_index, col_index] = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>return extrema, gradient<EOL>

Takes a spectrogram and returns a 2D array of the form:

0 0 0 1 0 0 1 0 0 0 1   <-- Frequency 0
0 0 1 0 0 0 0 0 0 1 0   <-- Frequency 1
0 0 0 0 0 0 1 0 1 0 0   <-- Frequency 2
*** Time axis *******

Where a 1 means that the value in that time bin in the spectrogram corresponds to
a peak/valley in the first derivative.

This function is used as part of the ASA algorithm and is not meant to be used publicly.


def _get_corresponding_offsets(onset_fronts, onset_front_id, onsets, offsets):

corresponding_offsets = []<EOL>for index in _get_front_idxs_from_id(onset_fronts, onset_front_id):<EOL><INDENT>offset_fidx, offset_sidx = _lookup_offset_by_onset_idx(index, onsets, offsets)<EOL>corresponding_offsets.append((offset_fidx, offset_sidx))<EOL><DEDENT>return corresponding_offsets<EOL>

Gets the offsets that occur as close as possible to the onsets in the given onset-front.


def _correlate_onsets_and_offsets(onsets, offsets, gradients):

<EOL>for freq_index, (ons, offs) in enumerate(zip(onsets[:, :], offsets[:, :])):<EOL><INDENT>indexes_of_all_ones = np.reshape(np.where(ons == <NUM_LIT:1>), (-<NUM_LIT:1>,))<EOL>last_idx = indexes_of_all_ones[<NUM_LIT:0>]<EOL>offs[<NUM_LIT:0>:last_idx + <NUM_LIT:1>] = <NUM_LIT:0><EOL>if len(indexes_of_all_ones > <NUM_LIT:1>):<EOL><INDENT>for next_idx in indexes_of_all_ones[<NUM_LIT:1>:]:<EOL><INDENT>offset_choices = offs[last_idx:next_idx]<EOL>offset_choice_indexes = np.where(offset_choices == <NUM_LIT:1>)<EOL>if not np.any(offset_choices):<EOL><INDENT>continue<EOL><DEDENT>assert np.any(offset_choices), "<STR_LIT>".format(last_idx, next_idx)<EOL>offset_choice_indexes = np.reshape(last_idx + offset_choice_indexes, (-<NUM_LIT:1>,))<EOL>assert np.all(offsets[freq_index, offset_choice_indexes])<EOL>gradient_values = gradients[freq_index, offset_choice_indexes]<EOL>index_of_largest_from_gradient_values = np.where(gradient_values == np.min(gradient_values))[<NUM_LIT:0>]<EOL>index_of_largest_offset_choice = offset_choice_indexes[index_of_largest_from_gradient_values]<EOL>assert offsets[freq_index, index_of_largest_offset_choice] == <NUM_LIT:1><EOL>offsets[freq_index, offset_choice_indexes] = <NUM_LIT:0><EOL>offsets[freq_index, index_of_largest_offset_choice] = <NUM_LIT:1><EOL>last_idx = next_idx<EOL><DEDENT><DEDENT>else:<EOL><INDENT>offsets[freq_index, :] = <NUM_LIT:0><EOL>offsets[freq_index, -<NUM_LIT:1>] = <NUM_LIT:1><EOL><DEDENT><DEDENT>return offsets<EOL>

Takes an array of onsets and an array of offsets, of the shape [nfrequencies, nsamples], where
each item in these arrays is either a 0 (not an on/offset) or a 1 (a possible on/offset).

This function returns a new offsets array, where there is a one-to-one correlation between
onsets and offsets, such that each onset has exactly one offset that occurs after it in
the time domain (the second dimension of the array).

The gradients array is used to decide which offset to use in the case of multiple possibilities.


def lowpass_filter(data, cutoff, fs, order=<NUM_LIT:5>):

nyq = <NUM_LIT:0.5> * fs<EOL>normal_cutoff = cutoff / nyq<EOL>b, a = signal.butter(order, normal_cutoff, btype='<STR_LIT>', analog=False)<EOL>y = signal.lfilter(b, a, data)<EOL>return y<EOL>

Does a lowpass filter over the given data.

:param data: The data (numpy array) to be filtered.
:param cutoff: The high cutoff in Hz.
:param fs: The sample rate in Hz of the data.
:param order: The order of the filter. The higher the order, the tighter the roll-off.
:returns: Filtered data (numpy array).


async def process_succeed_response(self, request, response):

pass<EOL>

Corresponding processing for the succeed response
:param request: Request
:param response: Response
:return:


def request(self,<EOL>url: str,<EOL>method: str = '<STR_LIT:GET>',<EOL>*,<EOL>callback=None,<EOL>encoding: typing.Optional[str] = None,<EOL>headers: dict = None,<EOL>metadata: dict = None,<EOL>request_config: dict = None,<EOL>request_session=None,<EOL>**kwargs):

headers = headers or {}<EOL>metadata = metadata or {}<EOL>request_config = request_config or {}<EOL>request_session = request_session or self.request_session<EOL>headers.update(self.headers.copy())<EOL>request_config.update(self.request_config.copy())<EOL>kwargs.update(self.kwargs.copy())<EOL>return Request(<EOL>url=url,<EOL>method=method,<EOL>callback=callback,<EOL>encoding=encoding,<EOL>headers=headers,<EOL>metadata=metadata,<EOL>request_config=request_config,<EOL>request_session=request_session,<EOL>**kwargs)<EOL>


async def _run_spider_hook(self, hook_func):

if callable(hook_func):<EOL><INDENT>try:<EOL><INDENT>aws_hook_func = hook_func(weakref.proxy(self))<EOL>if isawaitable(aws_hook_func):<EOL><INDENT>await aws_hook_func<EOL><DEDENT><DEDENT>except Exception as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT><DEDENT>

Run hook before/after spider start crawling
:param hook_func: aws function
:return:


def __init__(self,<EOL>middleware: typing.Union[typing.Iterable, Middleware] = None,<EOL>loop=None,<EOL>is_async_start: bool = False):

if not self.start_urls or not isinstance(self.start_urls,<EOL>collections.Iterable):<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT>self.callback_result_map = self.callback_result_map or {}<EOL>self.request_config = self.request_config or {}<EOL>self.headers = self.headers or {}<EOL>self.metadata = self.metadata or {}<EOL>self.kwargs = self.kwargs or {}<EOL>self.request_config = self.request_config or {}<EOL>self.is_async_start = is_async_start<EOL>self.loop = loop<EOL>asyncio.set_event_loop(self.loop)<EOL>if isinstance(middleware, list):<EOL><INDENT>self.middleware = reduce(lambda x, y: x + y, middleware)<EOL><DEDENT>else:<EOL><INDENT>self.middleware = middleware or Middleware()<EOL><DEDENT>self.request_queue = asyncio.Queue()<EOL>self.sem = asyncio.Semaphore(self.concurrency)<EOL>

Init spider object.
:param middleware: a list of or a single Middleware
:param loop:
:param is_async_start:


async def handle_request(self, request: Request<EOL>) -> typing.Tuple[AsyncGeneratorType, Response]:

callback_result, response = None, None<EOL>await self._run_request_middleware(request)<EOL>try:<EOL><INDENT>callback_result, response = await request.fetch_callback(self.sem)<EOL><DEDENT>except NotImplementedParseError as e:<EOL><INDENT>self.logger.error(e)<EOL><DEDENT>except NothingMatchedError as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT>except Exception as e:<EOL><INDENT>self.logger.error(f'<STR_LIT>')<EOL><DEDENT>await self._run_response_middleware(request, response)<EOL>await self._process_response(request=request, response=response)<EOL>return callback_result, response<EOL>

Wrap request with middleware.
:param request:
:return:


def __init__(self, default: str = '<STR_LIT>', many: bool = False):

self.default = default<EOL>self.many = many<EOL>

Init BaseField class
url: http://lxml.de/index.html
:param default: default value
:param many: if there are many fields in one page


def __init__(self,<EOL>css_select: str = None,<EOL>xpath_select: str = None,<EOL>default: str = None,<EOL>many: bool = False):

super(_LxmlElementField, self).__init__(default=default, many=many)<EOL>self.css_select = css_select<EOL>self.xpath_select = xpath_select<EOL>

:param css_select: css select http://lxml.de/cssselect.html
:param xpath_select: http://www.w3school.com.cn/xpath/index.asp
:param default: inherit
:param many: inherit


return str(value).strip()<EOL>


def django_boolean_icon(field_val, alt_text=None, title=None):

<EOL>BOOLEAN_MAPPING = {True: '<STR_LIT:yes>', False: '<STR_LIT>', None: '<STR_LIT>'}<EOL>alt_text = alt_text or BOOLEAN_MAPPING[field_val]<EOL>if title is not None:<EOL><INDENT>title = '<STR_LIT>' % title<EOL><DEDENT>else:<EOL><INDENT>title = '<STR_LIT>'<EOL><DEDENT>return mark_safe('<STR_LIT>' %<EOL>(settings.STATIC_URL, BOOLEAN_MAPPING[field_val], alt_text, title))<EOL>

Return HTML code for a nice representation of true/false.


def _fn(self, item):<EOL><INDENT>return ajax_editable_boolean_cell(item, attr)<EOL><DEDENT>_fn.allow_tags = True<EOL>_fn.short_description = short_description<EOL>_fn.editable_boolean_field = attr<EOL>return _fn<EOL>

Convenience function: Assign the return value of this method to a variable
of your ModelAdmin class and put the variable name into list_display.

Example::

    class MyTreeEditor(TreeEditor):
        list_display = ('__unicode__', 'active_toggle')

        active_toggle = ajax_editable_boolean('active', _('is active'))


def has_change_permission(self, request, obj=None):

if settings.TREE_EDITOR_OBJECT_PERMISSIONS:<EOL><INDENT>opts = self.opts<EOL>r = request.user.has_perm(opts.app_label + '<STR_LIT:.>' + opts.get_change_permission(), obj)<EOL><DEDENT>else:<EOL><INDENT>r = True<EOL><DEDENT>return r and super(TreeEditor, self).has_change_permission(request, obj)<EOL>

Implement a lookup for object level permissions. Basically the same as
ModelAdmin.has_change_permission, but also passes the obj parameter in.


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>self._ajax_editable_booleans = {}<EOL>for field in self.list_display:<EOL><INDENT>item = getattr(self.__class__, field, None)<EOL>if not item:<EOL><INDENT>continue<EOL><DEDENT>attr = getattr(item, '<STR_LIT>', None)<EOL>if attr:<EOL><INDENT>def _fn(self, instance):<EOL><INDENT>return [ajax_editable_boolean_cell(instance, _fn.attr)]<EOL><DEDENT>_fn.attr = attr<EOL>result_func = getattr(item, '<STR_LIT>', _fn)<EOL>self._ajax_editable_booleans[attr] = result_func<EOL><DEDENT><DEDENT>

Collect all fields marked as editable booleans. We do not
want the user to be able to edit arbitrary fields by crafting
an AJAX request by hand.