if not src_path.endswith('<STR_LIT:/>'):<EOL><INDENT>src_path += '<STR_LIT:/>'<EOL><DEDENT>if not dest_path.endswith('<STR_LIT:/>'):<EOL><INDENT>dest_path += '<STR_LIT:/>'<EOL><DEDENT>common_prefix = os.path.commonprefix([src_path, dest_path])<EOL>if common_prefix == src_path:<EOL><INDENT>msg = '<STR_LIT>'.format(<EOL>common_prefix, src_path)<EOL>raise RuntimeError(msg)<EOL><DEDENT>if common_prefix == dest_path:<EOL><INDENT>msg = '<STR_LIT>'.format(<EOL>common_prefix, dest_path)<EOL>raise RuntimeError(msg)<EOL><DEDENT>delete_dir(bucket_name, dest_path,<EOL>aws_access_key_id, aws_secret_access_key)<EOL>session = boto3.session.Session(<EOL>aws_access_key_id=aws_access_key_id,<EOL>aws_secret_access_key=aws_secret_access_key,<EOL>profile_name=aws_profile)<EOL>s3 = session.resource('<STR_LIT>')<EOL>bucket = s3.Bucket(bucket_name)<EOL>for src_obj in bucket.objects.filter(Prefix=src_path):<EOL><INDENT>src_rel_path = os.path.relpath(src_obj.key, start=src_path)<EOL>dest_key_path = os.path.join(dest_path, src_rel_path)<EOL>head = s3.meta.client.head_object(Bucket=bucket_name,<EOL>Key=src_obj.key)<EOL>metadata = head['<STR_LIT>']<EOL>content_type = head['<STR_LIT>']<EOL>if cache_control is None and '<STR_LIT>' in head:<EOL><INDENT>cache_control = head['<STR_LIT>']<EOL><DEDENT>if surrogate_control is not None:<EOL><INDENT>metadata['<STR_LIT>'] = surrogate_control<EOL><DEDENT>if surrogate_key is not None:<EOL><INDENT>metadata['<STR_LIT>'] = surrogate_key<EOL><DEDENT>s3.meta.client.copy_object(<EOL>Bucket=bucket_name,<EOL>Key=dest_key_path,<EOL>CopySource={'<STR_LIT>': bucket_name, '<STR_LIT>': src_obj.key},<EOL>MetadataDirective='<STR_LIT>',<EOL>Metadata=metadata,<EOL>ACL='<STR_LIT>',<EOL>CacheControl=cache_control,<EOL>ContentType=content_type)<EOL><DEDENT>if create_directory_redirect_object:<EOL><INDENT>dest_dirname = dest_path.rstrip('<STR_LIT:/>')<EOL>obj = bucket.Object(dest_dirname)<EOL>metadata = {'<STR_LIT>': '<STR_LIT:true>'}<EOL>obj.put(Body='<STR_LIT>',<EOL>ACL='<STR_LIT>',<EOL>Metadata=metadata,<EOL>CacheControl=cache_control)<EOL><DEDENT>

Copy objects from one directory in a bucket to another directory in
    the same bucket.

    Object metadata is preserved while copying, with the following exceptions:

    - If a new surrogate key is provided it will replace the original one.
    - If ``cache_control`` and ``surrogate_control`` values are provided they
      will replace the old one.

    Parameters
    ----------
    bucket_name : `str`
        Name of an S3 bucket.
    src_path : `str`
        Source directory in the S3 bucket. The ``src_path`` should ideally end
        in a trailing `'/'`. E.g. `'dir/dir2/'`.
    dest_path : `str`
        Destination directory in the S3 bucket. The ``dest_path`` should
        ideally end in a trailing `'/'`. E.g. `'dir/dir2/'`. The destination
        path cannot contain the source path.
    aws_access_key_id : `str`
        The access key for your AWS account. Also set
        ``aws_secret_access_key``.
    aws_secret_access_key : `str`
        The secret key for your AWS account.
    aws_profile : `str`, optional
        Name of AWS profile in :file:`~/.aws/credentials`. Use this instead
        of ``aws_access_key_id`` and ``aws_secret_access_key`` for file-based
        credentials.
    surrogate_key : `str`, optional
        The surrogate key to insert in the header of all objects in the
        ``x-amz-meta-surrogate-key`` field. This key is used to purge
        builds from the Fastly CDN when Editions change.
        If `None` then no header will be set.
        If the object already has a ``x-amz-meta-surrogate-key`` header then
        it will be replaced.
    cache_control : `str`, optional
        This sets (and overrides) the ``Cache-Control`` header on the copied
        files. The ``Cache-Control`` header specifically dictates how content
        is cached by the browser (if ``surrogate_control`` is also set).
    surrogate_control : `str`, optional
        This sets (and overrides) the ``x-amz-meta-surrogate-control`` header
        on the copied files. The ``Surrogate-Control``
        or ``x-amz-meta-surrogate-control`` header is used in priority by
        Fastly to givern it's caching. This caching policy is *not* passed
        to the browser.
    create_directory_redirect_object : `bool`, optional
        Create a directory redirect object for the root directory. The
        directory redirect object is an empty S3 object named after the
        directory (without a trailing slash) that contains a
        ``x-amz-meta-dir-redirect=true`` HTTP header. LSST the Docs' Fastly
        VCL is configured to redirect requests for a directory path to the
        directory's ``index.html`` (known as *courtesy redirects*).

    Raises
    ------
    ltdconveyor.s3.S3Error
        Thrown by any unexpected faults from the S3 API.
    RuntimeError
        Thrown when the source and destination directories are the same.


logger = logging.getLogger(__name__)<EOL>logger.info('<STR_LIT>', __name__)<EOL>if ctx.obj['<STR_LIT>'] is None:<EOL><INDENT>if ctx.obj['<STR_LIT:username>'] is None or ctx.obj['<STR_LIT:password>'] is None:<EOL><INDENT>raise click.UsageError(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>sys.exit(<NUM_LIT:1>)<EOL><DEDENT>logger.debug(<EOL>'<STR_LIT>',<EOL>ctx.obj['<STR_LIT:username>'],<EOL>ctx.obj['<STR_LIT>'])<EOL>token = get_keeper_token(<EOL>ctx.obj['<STR_LIT>'],<EOL>ctx.obj['<STR_LIT:username>'],<EOL>ctx.obj['<STR_LIT:password>'])<EOL>ctx.obj['<STR_LIT>'] = token<EOL>logger.debug(<EOL>'<STR_LIT>',<EOL>ctx.obj['<STR_LIT:username>'],<EOL>ctx.obj['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>logger.debug(<EOL>'<STR_LIT>')<EOL><DEDENT>

Ensure a token is in the Click context object or authenticate and obtain
    the token from LTD Keeper.

    Parameters
    ----------
    ctx : `click.Context`
        The Click context. ``ctx.obj`` must be a `dict` that contains keys:
        ``keeper_hostname``, ``username``, ``password``, ``token``. This
        context object is prepared by the main Click group,
        `ltdconveyor.cli.main.main`.


data = {<EOL>'<STR_LIT>': True<EOL>}<EOL>r = requests.patch(<EOL>build_url,<EOL>auth=(keeper_token, '<STR_LIT>'),<EOL>json=data)<EOL>if r.status_code != <NUM_LIT:200>:<EOL><INDENT>raise KeeperError(r)<EOL><DEDENT>

Confirm a build upload is complete.

    Wraps ``PATCH /builds/{build}``.

    Parameters
    ----------
    build_url : `str`
        URL of the build resource. Given a build resource, this URL is
        available from the ``self_url`` field.
    keeper_token : `str`
        Auth token (`ltdconveyor.keeper.get_keeper_token`).

    Raises
    ------
    ltdconveyor.keeper.KeeperError
        Raised if there is an error communicating with the LTD Keeper API.


val = item.get(source_name.encode('<STR_LIT:utf-8>'), None)<EOL>if val is not None:<EOL><INDENT>val = convert_string(val)<EOL><DEDENT>return val<EOL>

This method receives an item from the source and a source name,
and returns the text content for the `source_name` node.


return force_text(smart_str(item.findtext(source_name))).strip()<EOL>


<EOL>request = urllib2.Request(url)<EOL>request.add_header('<STR_LIT>', '<STR_LIT>')<EOL>opener = urllib2.build_opener()<EOL>response = opener.open(request)<EOL>data = response.read()<EOL>if response.headers.get('<STR_LIT>', '<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>stream = StringIO.StringIO(data)<EOL>gzipper = gzip.GzipFile(fileobj=stream)<EOL>data = gzipper.read()<EOL><DEDENT>f = open(dest, '<STR_LIT:wb>')<EOL>f.write(data)<EOL>f.close()<EOL>

Downloads a HTTP resource from `url` and save to `dest`.

Capable of dealing with Gzip compressed content.


@staticmethod<EOL><INDENT>def build(data_path: str) -> Path:<DEDENT>

return Path(data_path)<EOL>

Base method that interprets ``data_path`` argument.

        Args:
            data_path: path to the tsv-file containing erroneous and corrected words

        Returns:
            the same path as a :class:`~pathlib.Path` object


@staticmethod<EOL><INDENT>def read(data_path: str, *args, **kwargs) -> Dict[str, List[Tuple[str, str]]]:<DEDENT>

fname = TyposKartaslov.build(data_path)<EOL>with open(str(fname), newline='<STR_LIT>', encoding='<STR_LIT:utf8>') as csvfile:<EOL><INDENT>reader = csv.reader(csvfile, delimiter='<STR_LIT:;>')<EOL>next(reader)<EOL>res = [(mistake, correct) for correct, mistake, weight in reader]<EOL><DEDENT>return {'<STR_LIT:train>': res}<EOL>

Read train data for spelling corrections algorithms

        Args:
            data_path: path that needs to be interpreted with :meth:`~deeppavlov.dataset_readers.typos_reader.TyposKartaslov.build`

        Returns:
            train data to pass to a :class:`~deeppavlov.dataset_iterators.typos_iterator.TyposDatasetIterator`


@classmethod<EOL><INDENT>@overrides<EOL>def read(self, data_path: str, dialogs: bool = False) -> Dict[str, List]:<DEDENT>

required_files = (self._data_fname(dt) for dt in ('<STR_LIT:train>', '<STR_LIT>', '<STR_LIT:test>'))<EOL>if not all(Path(data_path, f).exists() for f in required_files):<EOL><INDENT>log.info('<STR_LIT>'.format(self.url, data_path))<EOL>download_decompress(self.url, data_path)<EOL>mark_done(data_path)<EOL><DEDENT>data = {<EOL>'<STR_LIT:train>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT:train>')), dialogs),<EOL>'<STR_LIT>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs),<EOL>'<STR_LIT:test>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT:test>')), dialogs)<EOL>}<EOL>return data<EOL>

Downloads ``'kvrest_public.tar.gz'``, decompresses, saves files to ``data_path``.

Parameters:
    data_path: path to save data
    dialogs: flag indices whether to output list of turns or list of dialogs

Returns:
    dictionary with ``'train'`` containing dialogs from ``'kvret_train_public.json'``, ``'valid'`` containing dialogs from ``'kvret_valid_public.json'``, ``'test'`` containing dialogs from ``'kvret_test_public.json'``. Each fields is a list of tuples ``(x_i, y_i)``.


required_files = (self._data_fname(dt) for dt in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>'))<EOL>if not all(Path(data_path, f).exists() for f in required_files):<EOL><INDENT>log.info('<STR_LIT>'.format(self.url, data_path))<EOL>download_decompress(self.url, data_path)<EOL>mark_done(data_path)<EOL><DEDENT>data = {<EOL>'<STR_LIT:train>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs),<EOL>'<STR_LIT>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs),<EOL>'<STR_LIT:test>': self._read_from_file(<EOL>Path(data_path, self._data_fname('<STR_LIT>')), dialogs)<EOL>}<EOL>return data<EOL>

Downloads ``'dstc2_v2.tar.gz'`` archive from ipavlov internal server,
decompresses and saves files to ``data_path``.

Parameters:
    data_path: path to save DSTC2 dataset
    dialogs: flag which indicates whether to output list of turns or
     list of dialogs

Returns:
    dictionary that contains ``'train'`` field with dialogs from
    ``'dstc2-trn.jsonlist'``, ``'valid'`` field with dialogs from
    ``'dstc2-val.jsonlist'`` and ``'test'`` field with dialogs from
    ``'dstc2-tst.jsonlist'``. Each field is a list of tuples ``(x_i, y_i)``.


def read(self, dir_path: str, dataset: Optional[str] = '<STR_LIT>', url: Optional[str] = None, *args, **kwargs)-> Dict[str, Dict[str, Any]]:

if url is not None:<EOL><INDENT>self.url = url<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_squad<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_sber_squad<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_multi_squad<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(dataset))<EOL><DEDENT>dir_path = Path(dir_path)<EOL>required_files = ['<STR_LIT>'.format(dt) for dt in ['<STR_LIT:train>', '<STR_LIT>']]<EOL>if not dir_path.exists():<EOL><INDENT>dir_path.mkdir()<EOL><DEDENT>if not all((dir_path / f).exists() for f in required_files):<EOL><INDENT>download_decompress(self.url, dir_path)<EOL><DEDENT>dataset = {}<EOL>for f in required_files:<EOL><INDENT>with dir_path.joinpath(f).open('<STR_LIT:r>', encoding='<STR_LIT:utf8>') as fp:<EOL><INDENT>data = json.load(fp)<EOL><DEDENT>if f == '<STR_LIT>':<EOL><INDENT>dataset['<STR_LIT>'] = data<EOL><DEDENT>else:<EOL><INDENT>dataset['<STR_LIT:train>'] = data<EOL><DEDENT><DEDENT>return dataset<EOL>

Args:
    dir_path: path to save data
    dataset: default dataset names: ``'SQuAD'``, ``'SberSQuAD'`` or ``'MultiSQuAD'``
    url: link to archive with dataset, use url argument if non-default dataset is used

Returns:
    dataset split on train/valid

Raises:
    RuntimeError: if `dataset` is not one of these: ``'SQuAD'``, ``'SberSQuAD'``, ``'MultiSQuAD'``.


def read(self, dir_path: str, dataset: Optional[str] = '<STR_LIT>', url: Optional[str] = None, *args,<EOL>**kwargs) -> Dict[str, Dict[str, Any]]:

if url is not None:<EOL><INDENT>self.url = url<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_multi_squad_retr<EOL><DEDENT>elif dataset == '<STR_LIT>':<EOL><INDENT>self.url = self.url_multi_squad_ru_retr<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>'.format(dataset))<EOL><DEDENT>dir_path = Path(dir_path)<EOL>required_files = ['<STR_LIT>'.format(dt) for dt in ['<STR_LIT:train>', '<STR_LIT>']]<EOL>if not dir_path.exists():<EOL><INDENT>dir_path.mkdir(parents=True)<EOL><DEDENT>if not all((dir_path / f).exists() for f in required_files):<EOL><INDENT>download_decompress(self.url, dir_path)<EOL><DEDENT>dataset = {}<EOL>for f in required_files:<EOL><INDENT>if '<STR_LIT>' in f:<EOL><INDENT>dataset['<STR_LIT>'] = dir_path.joinpath(f)<EOL><DEDENT>else:<EOL><INDENT>dataset['<STR_LIT:train>'] = dir_path.joinpath(f)<EOL><DEDENT><DEDENT>return dataset<EOL>

Args:
    dir_path: path to save data
    dataset: default dataset names: ``'MultiSQuADRetr'``, ``'MultiSQuADRuRetr'``
    url: link to archive with dataset, use url argument if non-default dataset is used

Returns:
    dataset split on train/valid

Raises:
    RuntimeError: if `dataset` is not one of these: ``'MultiSQuADRetr'``, ``'MultiSQuADRuRetr'``.


def read(self, data_path: Union[Path, str], db_url: Optional[str] = None, *args,<EOL>**kwargs) -> None:

logger.info('<STR_LIT>')<EOL>try:<EOL><INDENT>save_path = expand_path(kwargs['<STR_LIT>'])<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ConfigError(<EOL>f'<STR_LIT>')<EOL><DEDENT>if save_path.exists() and save_path.with_suffix(f'<STR_LIT>').exists():<EOL><INDENT>return<EOL><DEDENT>try:<EOL><INDENT>dataset_format = kwargs['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ConfigError(<EOL>f'<STR_LIT>')<EOL><DEDENT>save_path.parent.mkdir(parents=True, exist_ok=True)<EOL>if db_url:<EOL><INDENT>download_dir = save_path.parent<EOL>logger.info(f'<STR_LIT>')<EOL>download(download_dir, db_url, force_download=False)<EOL>return<EOL><DEDENT>self._build_db(save_path, dataset_format, expand_path(data_path))<EOL>

Build a SQLite database from provided files, download SQLite database from a provided URL,
         or do nothing.

        Args:
            data_path: a directory/file with texts to create a database from
            db_url: path to a database url
            kwargs:
                save_path: a path where a database should be saved to, or path to a ready database
                dataset_format: initial data format; should be selected from ['txt', 'wiki', 'json']

        Returns:
            None


try:<EOL><INDENT>predictions = [np.round(x) for x in y_predicted]<EOL><DEDENT>except TypeError:<EOL><INDENT>predictions = y_predicted<EOL><DEDENT>return f1_score(np.array(y_true), np.array(predictions), average="<STR_LIT>")<EOL>

Calculates F1 weighted measure.

Args:
    y_true: list of true values
    y_predicted: list of predicted values

Returns:
    F1 score


def compute_bleu(reference_corpus, translation_corpus, max_order=<NUM_LIT:4>,<EOL>smooth=False):

matches_by_order = [<NUM_LIT:0>] * max_order<EOL>possible_matches_by_order = [<NUM_LIT:0>] * max_order<EOL>reference_length = <NUM_LIT:0><EOL>translation_length = <NUM_LIT:0><EOL>for (references, translation) in zip(reference_corpus,<EOL>translation_corpus):<EOL><INDENT>reference_length += min(len(r) for r in references)<EOL>translation_length += len(translation)<EOL>merged_ref_ngram_counts = collections.Counter()<EOL>for reference in references:<EOL><INDENT>merged_ref_ngram_counts |= _get_ngrams(reference, max_order)<EOL><DEDENT>translation_ngram_counts = _get_ngrams(translation, max_order)<EOL>overlap = translation_ngram_counts & merged_ref_ngram_counts<EOL>for ngram in overlap:<EOL><INDENT>matches_by_order[len(ngram)-<NUM_LIT:1>] += overlap[ngram]<EOL><DEDENT>for order in range(<NUM_LIT:1>, max_order+<NUM_LIT:1>):<EOL><INDENT>possible_matches = len(translation) - order + <NUM_LIT:1><EOL>if possible_matches > <NUM_LIT:0>:<EOL><INDENT>possible_matches_by_order[order-<NUM_LIT:1>] += possible_matches<EOL><DEDENT><DEDENT><DEDENT>precisions = [<NUM_LIT:0>] * max_order<EOL>for i in range(<NUM_LIT:0>, max_order):<EOL><INDENT>if smooth:<EOL><INDENT>precisions[i] = ((matches_by_order[i] + <NUM_LIT:1.>) /<EOL>(possible_matches_by_order[i] + <NUM_LIT:1.>))<EOL><DEDENT>else:<EOL><INDENT>if possible_matches_by_order[i] > <NUM_LIT:0>:<EOL><INDENT>precisions[i] = (float(matches_by_order[i]) /<EOL>possible_matches_by_order[i])<EOL><DEDENT>else:<EOL><INDENT>precisions[i] = <NUM_LIT:0.0><EOL><DEDENT><DEDENT><DEDENT>if min(precisions) > <NUM_LIT:0>:<EOL><INDENT>p_log_sum = sum((<NUM_LIT:1.> / max_order) * math.log(p) for p in precisions)<EOL>geo_mean = math.exp(p_log_sum)<EOL><DEDENT>else:<EOL><INDENT>geo_mean = <NUM_LIT:0><EOL><DEDENT>ratio = float(translation_length) / reference_length<EOL>if ratio > <NUM_LIT:1.0>:<EOL><INDENT>bp = <NUM_LIT:1.><EOL><DEDENT>else:<EOL><INDENT>bp = math.exp(<NUM_LIT:1> - <NUM_LIT:1.> / ratio)<EOL><DEDENT>bleu = geo_mean * bp<EOL>return (bleu, precisions, bp, ratio, translation_length, reference_length)<EOL>

Computes BLEU score of translated segments against one or more references.

    Args:
      reference_corpus: list of lists of references for each translation. Each
          reference should be tokenized into a list of tokens.
      translation_corpus: list of translations to score. Each translation
          should be tokenized into a list of tokens.
      max_order: Maximum n-gram order to use when computing BLEU score.
      smooth: Whether or not to apply Lin et al. 2004 smoothing.

    Returns:
      3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram
      precisions and brevity penalty.


ngram_counts = collections.Counter()<EOL>for order in range(<NUM_LIT:1>, max_order + <NUM_LIT:1>):<EOL><INDENT>for i in range(<NUM_LIT:0>, len(segment) - order + <NUM_LIT:1>):<EOL><INDENT>ngram = tuple(segment[i:i+order])<EOL>ngram_counts[ngram] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return ngram_counts<EOL>

Extracts all n-grams upto a given maximum order from an input segment.

    Args:
      segment: text segment from which n-grams will be extracted.
      max_order: maximum length in tokens of the n-grams returned by this
          methods.

    Returns:
      The Counter containing all n-grams upto max_order in segment
      with a count of how many times each n-gram occurred.


@register_metric('<STR_LIT>')<EOL>def squad_v2_f1(y_true: List[List[str]], y_predicted: List[str]) -> float:

f1_total = <NUM_LIT:0.0><EOL>for ground_truth, prediction in zip(y_true, y_predicted):<EOL><INDENT>prediction_tokens = normalize_answer(prediction).split()<EOL>f1s = []<EOL>for gt in ground_truth:<EOL><INDENT>gt_tokens = normalize_answer(gt).split()<EOL>if len(gt_tokens) == <NUM_LIT:0> or len(prediction_tokens) == <NUM_LIT:0>:<EOL><INDENT>f1s.append(float(gt_tokens == prediction_tokens))<EOL>continue<EOL><DEDENT>common = Counter(prediction_tokens) & Counter(gt_tokens)<EOL>num_same = sum(common.values())<EOL>if num_same == <NUM_LIT:0>:<EOL><INDENT>f1s.append(<NUM_LIT:0.0>)<EOL>continue<EOL><DEDENT>precision = <NUM_LIT:1.0> * num_same / len(prediction_tokens)<EOL>recall = <NUM_LIT:1.0> * num_same / len(gt_tokens)<EOL>f1 = (<NUM_LIT:2> * precision * recall) / (precision + recall)<EOL>f1s.append(f1)<EOL><DEDENT>f1_total += max(f1s)<EOL><DEDENT>return <NUM_LIT:100> * f1_total / len(y_true) if len(y_true) > <NUM_LIT:0> else <NUM_LIT:0><EOL>

Calculates F-1 score between y_true and y_predicted
        F-1 score uses the best matching y_true answer

    The same as in SQuAD-v2.0

    Args:
        y_true: list of correct answers (correct answers are represented by list of strings)
        y_predicted: list of predicted answers

    Returns:
        F-1 score : float


def fit(self, data: List[Dict[Any, Any]]) -> None:

log.info(f"<STR_LIT>")<EOL>self.ec_data = [dict(item, **{<EOL>'<STR_LIT>': self.preprocess.spacy2dict(self.preprocess.analyze(item['<STR_LIT>'])),<EOL>'<STR_LIT>': self.preprocess.spacy2dict(self.preprocess.analyze(item['<STR_LIT>']+'<STR_LIT>'+item['<STR_LIT>']))<EOL>}) for item in data]<EOL>log.info('<STR_LIT>')<EOL>

Preprocess items `title` and `description` from the `data`

        Parameters:
            data: list of catalog items

        Returns:
            None


log.info(f"<STR_LIT>")<EOL>for path in self.load_path:<EOL><INDENT>if Path.is_file(path):<EOL><INDENT>self.ec_data += load_pickle(path)<EOL><DEDENT>else:<EOL><INDENT>raise FileNotFoundError<EOL><DEDENT><DEDENT>log.info(f"<STR_LIT>")<EOL>


log.info("<STR_LIT>".format(self.load_path))<EOL>self.ec_data, self.x_train_features = load_pickle(<EOL>expand_path(self.load_path))<EOL>


log.info("<STR_LIT>".format(self.save_path))<EOL>path = expand_path(self.save_path)<EOL>save_pickle((self.ec_data, self.x_train_features), path)<EOL>


def __call__(self, utterances_batch: list, history_batch: list,<EOL>states_batch: Optional[list]=None) -> Tuple[list, list, list]:

batch_len = len(utterances_batch)<EOL>confidence_batch = [<NUM_LIT:1.0>] * batch_len<EOL>response_batch: List[Optional[str]] = [None] * batch_len<EOL>infer_indexes = []<EOL>if not states_batch:<EOL><INDENT>states_batch: List[Optional[dict]] = [None] * batch_len<EOL><DEDENT>for utt_i, utterance in enumerate(utterances_batch):<EOL><INDENT>if not states_batch[utt_i]:<EOL><INDENT>states_batch[utt_i] = {'<STR_LIT>': list(self.model.in_x), '<STR_LIT>': []}<EOL><DEDENT>if utterance:<EOL><INDENT>states_batch[utt_i]['<STR_LIT>'].pop(<NUM_LIT:0>)<EOL>states_batch[utt_i]['<STR_LIT>'].append(utterance)<EOL><DEDENT>if states_batch[utt_i]['<STR_LIT>']:<EOL><INDENT>response = self.proposal.format(states_batch[utt_i]['<STR_LIT>'][<NUM_LIT:0>])<EOL>response_batch[utt_i] = response<EOL><DEDENT>else:<EOL><INDENT>infer_indexes.append(utt_i)<EOL><DEDENT><DEDENT>if infer_indexes:<EOL><INDENT>infer_utterances = zip(*[tuple(states_batch[i]['<STR_LIT>']) for i in infer_indexes])<EOL>infer_results = self.model(*infer_utterances)<EOL>if len(self.model.out_params) > <NUM_LIT:1>:<EOL><INDENT>infer_results = ['<STR_LIT>'.join([str(out_y) for out_y in result]) for result in zip(*infer_results)]<EOL><DEDENT>for infer_i, infer_result in zip(infer_indexes, infer_results):<EOL><INDENT>response_batch[infer_i] = infer_result<EOL>states_batch[infer_i] = None<EOL><DEDENT><DEDENT>return response_batch, confidence_batch, states_batch<EOL>

Returns skill inference result.

        Returns batches of skill inference results, estimated confidence
            levels and up to date states corresponding to incoming utterance
            batch. Also handles interaction with multiargument models using
            skill states.

        Args:
            utterances_batch: A batch of utterances of any type.
            history_batch: Not used. A batch of list typed histories for each
                utterance.
            states_batch: A batch of states for each utterance.

        Returns:
            response: A batch of arbitrary typed skill inference results.
            confidence: A batch of float typed confidence levels for each of
                skill inference result.
            states: Optional. A batch of states for each response.


unit = np.eye(k, dtype=int)<EOL>return unit[x]<EOL>

Takes an array of integers and transforms it
to an array of one-hot encoded vectors


@property<EOL><INDENT>def tags_number_(self) -> int:<DEDENT>

return len(self.tags)<EOL>


def next_generation(self, generation: List[dict], scores: List[float], iteration: int) -> List[dict]:

next_population = self.selection_of_best_with_weights(generation, scores)<EOL>log.info("<STR_LIT>".format(self.n_saved_best_pretrained))<EOL>offsprings = self.crossover(generation, scores)<EOL>changable_next = self.mutation(offsprings)<EOL>next_population.extend(changable_next)<EOL>for i in range(self.n_saved_best_pretrained):<EOL><INDENT>if self.train_partition != <NUM_LIT:1>:<EOL><INDENT>file_ext = str(Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).suffix)<EOL>next_population[i]["<STR_LIT>"]["<STR_LIT:train>"] = "<STR_LIT:_>".join(<EOL>Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).stem.split("<STR_LIT:_>")[:-<NUM_LIT:1>]<EOL>) + "<STR_LIT:_>" + str(iteration % self.train_partition) + file_ext<EOL><DEDENT>if self.elitism_with_weights:<EOL><INDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_load_path,<EOL>self.get_value_from_config(next_population[i], self.path_to_models_save_path))<EOL><DEDENT>else:<EOL><INDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_load_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL><DEDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_save_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL><DEDENT>for i in range(self.n_saved_best_pretrained, self.population_size):<EOL><INDENT>if self.train_partition != <NUM_LIT:1>:<EOL><INDENT>file_ext = str(Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).suffix)<EOL>next_population[i]["<STR_LIT>"]["<STR_LIT:train>"] = "<STR_LIT:_>".join(<EOL>[str(p) for p in Path(next_population[i]["<STR_LIT>"]["<STR_LIT:train>"]).stem.split("<STR_LIT:_>")[:-<NUM_LIT:1>]])+ "<STR_LIT:_>" + str(iteration % self.train_partition) + file_ext<EOL><DEDENT>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_save_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL>self.insert_value_or_dict_into_config(<EOL>next_population[i], self.path_to_models_load_path,<EOL>str(self.models_path / f"<STR_LIT>" / f"<STR_LIT>"))<EOL>next_population[i]["<STR_LIT>"] = self.evolution_model_id<EOL>self.evolution_model_id += <NUM_LIT:1><EOL><DEDENT>return next_population<EOL>

Provide replacement

Args:
    generation: current generation (set of self.population_size configs
    scores: corresponding scores that should be maximized
    iteration: iteration number

Returns:
    the next generation according to the given scores of current generation


def crossover(self, population: List[dict], scores: List[float]) -> List[dict]:

offsprings = []<EOL>ranges = self.range_scores(scores)<EOL>a = <NUM_LIT:1.> / (<NUM_LIT:1.> - self.population_size)<EOL>b = self.population_size / (self.population_size - <NUM_LIT:1.>)<EOL>probas_to_be_parent = (a * ranges + b) / np.sum(a * ranges + b)<EOL>intervals = np.array([np.sum(probas_to_be_parent[:i]) for i in range(self.population_size)])<EOL>for i in range(self.population_size - self.n_saved_best_pretrained):<EOL><INDENT>rs = np.random.random(<NUM_LIT:2>)<EOL>parents = population[np.where(rs[<NUM_LIT:0>] > intervals)[<NUM_LIT:0>][-<NUM_LIT:1>]], population[np.where(rs[<NUM_LIT:1>] > intervals)[<NUM_LIT:0>][-<NUM_LIT:1>]]<EOL>if self.decision(self.p_crossover):<EOL><INDENT>params_perm = np.random.permutation(self.n_params)<EOL>curr_offsprings = [deepcopy(parents[<NUM_LIT:0>]),<EOL>deepcopy(parents[<NUM_LIT:1>])]<EOL>part = int(self.crossover_power * self.n_params)<EOL>for j in range(self.n_params - part, self.n_params):<EOL><INDENT>self.insert_value_or_dict_into_config(curr_offsprings[<NUM_LIT:0>],<EOL>self.paths_to_params[<EOL>params_perm[j]],<EOL>self.get_value_from_config(<EOL>parents[<NUM_LIT:1>],<EOL>self.paths_to_params[<EOL>params_perm[j]]))<EOL>self.insert_value_or_dict_into_config(curr_offsprings[<NUM_LIT:1>],<EOL>self.paths_to_params[<EOL>params_perm[j]],<EOL>self.get_value_from_config(<EOL>parents[<NUM_LIT:0>],<EOL>self.paths_to_params[<EOL>params_perm[j]]))<EOL><DEDENT>offsprings.append(deepcopy(curr_offsprings[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>offsprings.append(deepcopy(parents[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT>return offsprings<EOL>

Recombine randomly population in pairs and cross over them with given probability.
Cross over from two parents produces two offsprings
each of which contains crossover_power portion of the parameter values from one parent,
 and the other (1 - crossover_power portion) from the other parent

Args:
    population: self.population_size individuums
    scores: list of corresponding scores

Returns:
    (self.population_size - self.n_saved_best_pretained) offsprings


def __init__(self, load_path: str, wiki_filename: str, entities_filename: str, inverted_index_filename: str,<EOL>id_to_name_file: str, lemmatize: bool = True, debug: bool = False, rule_filter_entities: bool = True,<EOL>use_inverted_index: bool = True, language: str = '<STR_LIT>', *args, **kwargs) -> None:

super().__init__(save_path=None, load_path=load_path)<EOL>self.morph = pymorphy2.MorphAnalyzer()<EOL>self.lemmatize = lemmatize<EOL>self.debug = debug<EOL>self.rule_filter_entities = rule_filter_entities<EOL>self.use_inverted_index = use_inverted_index<EOL>self._language = language<EOL>if language not in self.LANGUAGES:<EOL><INDENT>log.warning(f'<STR_LIT>')<EOL><DEDENT>self._wiki_filename = wiki_filename<EOL>self._entities_filename = entities_filename<EOL>self.inverted_index_filename = inverted_index_filename<EOL>self.id_to_name_file = id_to_name_file<EOL>self.name_to_q: Optional[Dict[str, List[Tuple[str]]]] = None<EOL>self.wikidata: Optional[Dict[str, List[List[str]]]] = None<EOL>self.inverted_index: Optional[Dict[str, List[Tuple[str]]]] = None<EOL>self.id_to_name: Optional[Dict[str, Dict[List[str]]]] = None<EOL>self.load()<EOL>if self.use_inverted_index:<EOL><INDENT>alphabet = "<STR_LIT>"<EOL>dictionary_words = list(self.inverted_index.keys())<EOL>self.searcher = LevenshteinSearcher(alphabet, dictionary_words)<EOL><DEDENT>

Args:
    load_path: path to folder with wikidata files
    wiki_filename: file with Wikidata triplets
    entities_filename: file with dict of entity titles (keys) and entity ids (values)
    inverted_index_filename: file with dict of words (keys) and entities containing these words (values)
    id_to_name_file: file with dict of entity ids (keys) and entities names and aliases (values)
    lemmatize: whether to lemmatize tokens of extracted entity
    debug: whether to print entities extracted from Wikidata
    rule_filter_entities: whether to filter entities which do not fit the question
    use_inverted_index: whether to use inverted index for entity linking
    language - the language of the linker (used for filtration of some questions to improve overall performance)
    *args:
    **kwargs:


def __call__(self, batch: List[List[List[Tuple[float, str]]]]) -> List[List[str]]:

return [self._infer_instance(candidates) for candidates in batch]<EOL>

Choose the best candidate for every token

        Args:
            batch: batch of probabilities and string values of candidates for every token in a sentence

        Returns:
            batch of corrected tokenized sentences


def transduce(self, first, second, threshold):

add_pred = (lambda x, y: x <= threshold)<EOL>clear_pred =(lambda x, y: False)<EOL>update_func = (lambda x, y: min(x, y))<EOL>costs, backtraces = self._fill_levenshtein_table(first, second,<EOL>update_func, add_pred, clear_pred,<EOL>threshold=threshold)<EOL>result = self._backtraces_to_transductions(first, second,<EOL>backtraces, threshold, return_cost=True)<EOL>return result<EOL>

Возвращает все трансдукции, переводящие first в second,
чья стоимость не превышает threshold

Возвращает:
----------
result : list
    список вида [(трансдукция, стоимость)]


def lower_transductions(self, word, max_cost, return_cost=True):

prefixes = [[] for i in range(len(word) + <NUM_LIT:1>)]<EOL>prefixes[<NUM_LIT:0>].append(((), <NUM_LIT:0.0>))<EOL>for pos in range(len(prefixes)):<EOL><INDENT>prefixes[pos] = self._perform_insertions(prefixes[pos], max_cost)<EOL>max_upperside_length = min(len(word) - pos, self.max_up_length)<EOL>for upperside_length in range(<NUM_LIT:1>, max_upperside_length + <NUM_LIT:1>):<EOL><INDENT>up = word[pos: pos + upperside_length]<EOL>for low, low_cost in self.operation_costs.get(up, dict()).items():<EOL><INDENT>for transduction, cost in prefixes[pos]:<EOL><INDENT>new_cost = cost + low_cost<EOL>if new_cost <= max_cost:<EOL><INDENT>new_transduction = transduction +(up, low)<EOL>prefixes[pos + upperside_length].append((new_transduction, new_cost))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>answer = sorted(prefixes[-<NUM_LIT:1>], key=(lambda x: x[<NUM_LIT:0>]))<EOL>if return_cost:<EOL><INDENT>return answer<EOL><DEDENT>else:<EOL><INDENT>return [elem[<NUM_LIT:0>] for elem in answer]<EOL><DEDENT>

Возвращает все трансдукции с верхним элементом word,
    чья стоимость не превышает max_cost

`   Возвращает:
    ----------
    result : list
        список вида [(трансдукция, стоимость)], если return_cost=True
        список трансдукций, если return_cost=False
        список отсортирован в порядке возрастания стоимости трансдукции


if s == "<STR_LIT>":<EOL><INDENT>return curr<EOL><DEDENT>curr_cash = self._descendance_cash[curr]<EOL>answer = curr_cash.get(s, None)<EOL>if answer is not None:<EOL><INDENT>return answer<EOL><DEDENT>res = curr<EOL>for a in s:<EOL><INDENT>res = self.graph[res][self.alphabet_codes[a]]<EOL>if res == Trie.NO_NODE:<EOL><INDENT>break<EOL><DEDENT><DEDENT>curr_cash[s] = res<EOL>return res<EOL>

Спуск из вершины curr по строке s с кэшированием


def find_partitions(self, s, max_count=<NUM_LIT:1>):

curr_agenda = [(self.root, [], <NUM_LIT:0>)]<EOL>for i, a in enumerate(s):<EOL><INDENT>next_agenda = []<EOL>for curr, borders, cost in curr_agenda:<EOL><INDENT>if cost >= max_count:<EOL><INDENT>continue<EOL><DEDENT>child = self.graph[curr][self.alphabet_codes[a]]<EOL>if child == Trie.NO_NODE:<EOL><INDENT>continue<EOL><DEDENT>next_agenda.append((child, borders, cost))<EOL>if self.is_final(child):<EOL><INDENT>next_agenda.append((self.root, borders + [i+<NUM_LIT:1>], cost+<NUM_LIT:1>))<EOL><DEDENT><DEDENT>curr_agenda = next_agenda<EOL><DEDENT>answer = []<EOL>for curr, borders, cost in curr_agenda:<EOL><INDENT>if curr == self.root:<EOL><INDENT>borders = [<NUM_LIT:0>] + borders<EOL>answer.append([s[left:borders[i+<NUM_LIT:1>]] for i, left in enumerate(borders[:-<NUM_LIT:1>])])<EOL><DEDENT><DEDENT>return answer<EOL>

Находит все разбиения s = s_1 ... s_m на словарные слова s_1, ..., s_m
для m <= max_count


if self.is_terminated:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>if s == "<STR_LIT>":<EOL><INDENT>self._set_final(self.root)<EOL>return<EOL><DEDENT>curr = self.root<EOL>for i, a in enumerate(s):<EOL><INDENT>code = self.alphabet_codes[a]<EOL>next = self.graph[curr][code]<EOL>if next == Trie.NO_NODE:<EOL><INDENT>curr = self._add_descendant(curr, s[i:])<EOL>break<EOL><DEDENT>else:<EOL><INDENT>curr = next<EOL><DEDENT><DEDENT>self._set_final(curr)<EOL>return self<EOL>

Добавление строки s в префиксный бор


self.final[curr] = True<EOL>

Делает состояние curr завершающим


branch, word, indexes = [self.root], [], [<NUM_LIT:0>]<EOL>letters_with_children = [self._get_children_and_letters(self.root)]<EOL>while len(branch) > <NUM_LIT:0>:<EOL><INDENT>if self.is_final(branch[-<NUM_LIT:1>]):<EOL><INDENT>yield "<STR_LIT>".join(word)<EOL><DEDENT>while indexes[-<NUM_LIT:1>] == len(letters_with_children[-<NUM_LIT:1>]):<EOL><INDENT>indexes.pop()<EOL>letters_with_children.pop()<EOL>branch.pop()<EOL>if len(indexes) == <NUM_LIT:0>:<EOL><INDENT>raise StopIteration()<EOL><DEDENT>word.pop()<EOL><DEDENT>next_letter, next_child = letters_with_children[-<NUM_LIT:1>][indexes[-<NUM_LIT:1>]]<EOL>indexes[-<NUM_LIT:1>] += <NUM_LIT:1><EOL>indexes.append(<NUM_LIT:0>)<EOL>word.append(next_letter)<EOL>branch.append(next_child)<EOL>letters_with_children.append(self._get_children_and_letters(branch[-<NUM_LIT:1>]))<EOL><DEDENT>

Возвращает итератор по словам, содержащимся в боре


def process_word(word: str, to_lower: bool = False,<EOL>append_case: Optional[str] = None) -> Tuple[str]:

if all(x.isupper() for x in word) and len(word) > <NUM_LIT:1>:<EOL><INDENT>uppercase = "<STR_LIT>"<EOL><DEDENT>elif word[<NUM_LIT:0>].isupper():<EOL><INDENT>uppercase = "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>uppercase = None<EOL><DEDENT>if to_lower:<EOL><INDENT>word = word.lower()<EOL><DEDENT>if word.isdigit():<EOL><INDENT>answer = ["<STR_LIT>"]<EOL><DEDENT>elif word.startswith("<STR_LIT>") or word.startswith("<STR_LIT>"):<EOL><INDENT>answer = ["<STR_LIT>"]<EOL><DEDENT>else:<EOL><INDENT>answer = list(word)<EOL><DEDENT>if to_lower and uppercase is not None:<EOL><INDENT>if append_case == "<STR_LIT>":<EOL><INDENT>answer = [uppercase] + answer<EOL><DEDENT>elif append_case == "<STR_LIT>":<EOL><INDENT>answer = answer + [uppercase]<EOL><DEDENT><DEDENT>return tuple(answer)<EOL>

Converts word to a tuple of symbols, optionally converts it to lowercase
    and adds capitalization label.

    Args:
        word: input word
        to_lower: whether to lowercase
        append_case: whether to add case mark
            ('<FIRST_UPPER>' for first capital and '<ALL_UPPER>' for all caps)

    Returns:
        a preprocessed word


for e in (el, el.lower(), el.capitalize(), el.upper()):<EOL><INDENT>if e in self.token2idx_dict:<EOL><INDENT>return self.token2idx_dict[e]<EOL><DEDENT><DEDENT>return <NUM_LIT:1><EOL>

Returns idx for el (token or char).

        Args:
            el: token or character

        Returns:
            idx in vocabulary


def __call__(self, contexts_raw: Tuple[str, ...], questions_raw: Tuple[str, ...],<EOL>**kwargs) -> Tuple[<EOL>List[str], List[List[str]], List[List[List[str]]],<EOL>List[List[int]], List[List[int]],<EOL>List[str], List[List[str]], List[List[List[str]]],<EOL>List[List[Tuple[int, int]]]<EOL>]:

contexts = []<EOL>contexts_tokens = []<EOL>contexts_chars = []<EOL>contexts_r2p = []<EOL>contexts_p2r = []<EOL>questions = []<EOL>questions_tokens = []<EOL>questions_chars = []<EOL>spans = []<EOL>for c_raw, q_raw in zip(contexts_raw, questions_raw):<EOL><INDENT>c, r2p, p2r = SquadPreprocessor.preprocess_str(c_raw, return_mapping=True)<EOL>c_tokens = [token.replace("<STR_LIT>", '<STR_LIT:">').replace("<STR_LIT>", '<STR_LIT:">') for token in word_tokenize(c)][:self.context_limit]<EOL>c_chars = [list(token)[:self.char_limit] for token in c_tokens]<EOL>q = SquadPreprocessor.preprocess_str(q_raw)<EOL>q_tokens = [token.replace("<STR_LIT>", '<STR_LIT:">').replace("<STR_LIT>", '<STR_LIT:">') for token in word_tokenize(q)][:self.question_limit]<EOL>q_chars = [list(token)[:self.char_limit] for token in q_tokens]<EOL>contexts.append(c)<EOL>contexts_tokens.append(c_tokens)<EOL>contexts_chars.append(c_chars)<EOL>contexts_r2p.append(r2p)<EOL>contexts_p2r.append(p2r)<EOL>questions.append(q)<EOL>questions_tokens.append(q_tokens)<EOL>questions_chars.append(q_chars)<EOL>spans.append(SquadPreprocessor.convert_idx(c, c_tokens))<EOL><DEDENT>return contexts, contexts_tokens, contexts_chars, contexts_r2p, contexts_p2r,questions, questions_tokens, questions_chars, spans<EOL>

Performs preprocessing of context and question
        Args:
            contexts_raw: batch of contexts to preprocess
            questions_raw: batch of questions to preprocess

        Returns:
            context: batch of processed contexts
            contexts_tokens: batch of tokenized contexts
            contexts_chars: batch of tokenized and split on chars contexts
            contexts_r2p: batch of mappings from raw context to processed context
            contexts_p2r: batch of mappings from procesesd context to raw context
            questions: batch of processed questions
            questions_tokens: batch of tokenized questions
            questions_chars: batch of tokenized and split on chars questions
            spans: batch of mapping tokens to position in context


def __call__(self, batch_docs: List[Union[str, List[str]]]) ->List[Union[List[str], List[List[str]]]]:

result = []<EOL>for docs in batch_docs:<EOL><INDENT>batch_chunks = []<EOL>if isinstance(docs, str):<EOL><INDENT>docs = [docs]<EOL><DEDENT>for doc in docs:<EOL><INDENT>if self.paragraphs:<EOL><INDENT>split_doc = doc.split('<STR_LIT>')<EOL>split_doc = [sd.strip() for sd in split_doc]<EOL>split_doc = list(filter(lambda x: len(x) > <NUM_LIT>, split_doc))<EOL>batch_chunks.append(split_doc)<EOL><DEDENT>else:<EOL><INDENT>doc_chunks = []<EOL>if self.keep_sentences:<EOL><INDENT>sentences = sent_tokenize(doc)<EOL>n_tokens = <NUM_LIT:0><EOL>keep = []<EOL>for s in sentences:<EOL><INDENT>n_tokens += len(s.split())<EOL>if n_tokens > self.tokens_limit:<EOL><INDENT>if keep:<EOL><INDENT>doc_chunks.append('<STR_LIT:U+0020>'.join(keep))<EOL>n_tokens = <NUM_LIT:0><EOL>keep.clear()<EOL><DEDENT><DEDENT>keep.append(s)<EOL><DEDENT>if keep:<EOL><INDENT>doc_chunks.append('<STR_LIT:U+0020>'.join(keep))<EOL><DEDENT>batch_chunks.append(doc_chunks)<EOL><DEDENT>else:<EOL><INDENT>split_doc = doc.split()<EOL>doc_chunks = [split_doc[i:i + self.tokens_limit] for i in<EOL>range(<NUM_LIT:0>, len(split_doc), self.tokens_limit)]<EOL>batch_chunks.append(doc_chunks)<EOL><DEDENT><DEDENT><DEDENT>result.append(batch_chunks)<EOL><DEDENT>if self.flatten_result:<EOL><INDENT>if isinstance(result[<NUM_LIT:0>][<NUM_LIT:0>], list):<EOL><INDENT>for i in range(len(result)):<EOL><INDENT>flattened = list(chain.from_iterable(result[i]))<EOL>result[i] = flattened<EOL><DEDENT><DEDENT><DEDENT>return result<EOL>

Make chunks from a batch of documents. There can be several documents in each batch.
        Args:
            batch_docs: a batch of documents / a batch of lists of documents
        Returns:
            chunks of docs, flattened or not


def __call__(self, texts_a: List[str], texts_b: Optional[List[str]] = None) -> List[InputFeatures]:

if texts_b is None:<EOL><INDENT>texts_b = [None] * len(texts_a)<EOL><DEDENT>examples = [InputExample(unique_id=<NUM_LIT:0>, text_a=text_a, text_b=text_b)<EOL>for text_a, text_b in zip(texts_a, texts_b)]<EOL>return convert_examples_to_features(examples, self.max_seq_length, self.tokenizer)<EOL>

Call Bert convert_examples_to_features function to tokenize and create masks.

        texts_a and texts_b are separated by [SEP] token

        Args:
            texts_a: list of texts,
            texts_b: list of texts, it could be None, e.g. single sentence classification task

        Returns:
            batch of InputFeatures with subtokens, subtoken ids, subtoken mask, segment mask.


def __call__(self, tokens_batch, **kwargs):

lemma_batch = []<EOL>for utterance in tokens_batch:<EOL><INDENT>lemma_utterance = []<EOL>for token in utterance:<EOL><INDENT>p = self.lemmatizer.parse(token)[<NUM_LIT:0>]<EOL>lemma_utterance.append(p.normal_form)<EOL><DEDENT>lemma_batch.append(lemma_utterance)<EOL><DEDENT>return lemma_batch<EOL>

Takes batch of tokens and returns the lemmatized tokens.


def parse_input(self, inp: str) -> Dict[Any, Any]:

state: List = []<EOL>for i in range(len(inp.split()) // <NUM_LIT:2>, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>state.append([inp.split(None, <NUM_LIT:1>)[<NUM_LIT:0>], inp.split(None, <NUM_LIT:1>)[<NUM_LIT:1>].split()[<NUM_LIT:0>]])<EOL>if i > <NUM_LIT:1>:<EOL><INDENT>inp = inp.split(None, <NUM_LIT:2>)[<NUM_LIT:2>]<EOL><DEDENT><DEDENT>return dict(state)<EOL>

Convert space-delimited string into dialog state


def spacy2dict(self, doc: spacy.tokens.Doc, fields: List[str] = None) -> List[Dict[Any, Any]]:

if fields is None:<EOL><INDENT>fields = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:text>']<EOL><DEDENT>return [{field: getattr(token, field) for field in fields} for token in doc]<EOL>

Convert SpaCy doc into list of tokens with `fields` properties only


def load(self, fname: str = None) -> None:

if fname is None:<EOL><INDENT>fname = self.load_path<EOL><DEDENT>fname = Path(fname).with_suffix('<STR_LIT>')<EOL>if fname.exists():<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class, str(fname)))<EOL>with open(fname, "<STR_LIT:rb>") as f:<EOL><INDENT>self.model = pickle.load(f)<EOL><DEDENT>warm_start = self.model_params.get("<STR_LIT>", None)<EOL>self.model_params = {param: getattr(self.model, param) for param in self.get_class_attributes(self.model)}<EOL>self.model_class = self.model.__module__ + self.model.__class__.__name__<EOL>log.info("<STR_LIT>".format(self.model_class))<EOL>if warm_start and "<STR_LIT>" in self.model_params.keys():<EOL><INDENT>self.model_params["<STR_LIT>"] = True<EOL>log.info("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>log.warning("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.warning("<STR_LIT>".format(str(fname)))<EOL>self.init_from_scratch()<EOL><DEDENT>return<EOL>

Initialize ``self.model`` as some sklearn model from saved re-initializing ``self.model_params`` parameters. \
    If in new given parameters ``warm_start`` is set to True and given model admits ``warm_start`` parameter, \
    model will be initilized from saved with opportunity to continue fitting.

Args:
    fname: string name of path to model to load from

Returns:
    None


@staticmethod<EOL><INDENT>def compose_input_data(x: List[Union[Tuple[Union[np.ndarray, list, spmatrix, str]],<EOL>List[Union[np.ndarray, list, spmatrix, str]],<EOL>np.ndarray, spmatrix]]) -> Union[spmatrix, np.ndarray]:<DEDENT>

x_features = []<EOL>for i in range(len(x)):<EOL><INDENT>if ((isinstance(x[i], tuple) or isinstance(x[i], list) or isinstance(x[i], np.ndarray) and len(x[i]))<EOL>or (issparse(x[i]) and x[i].shape[<NUM_LIT:0>])):<EOL><INDENT>if issparse(x[i][<NUM_LIT:0>]):<EOL><INDENT>x_features.append(vstack(list(x[i])))<EOL><DEDENT>elif isinstance(x[i][<NUM_LIT:0>], np.ndarray) or isinstance(x[i][<NUM_LIT:0>], list):<EOL><INDENT>x_features.append(np.vstack(list(x[i])))<EOL><DEDENT>elif isinstance(x[i][<NUM_LIT:0>], str):<EOL><INDENT>x_features.append(np.array(x[i]))<EOL><DEDENT>else:<EOL><INDENT>raise ConfigError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT><DEDENT>sparse = False<EOL>for inp in x_features:<EOL><INDENT>if issparse(inp):<EOL><INDENT>sparse = True<EOL><DEDENT><DEDENT>if sparse:<EOL><INDENT>x_features = hstack(list(x_features))<EOL><DEDENT>else:<EOL><INDENT>x_features = np.hstack(list(x_features))<EOL><DEDENT>return x_features<EOL>

Stack given list of different types of inputs to the one matrix. If one of the inputs is a sparse matrix, \
    then output will be also a sparse matrix

Args:
    x: list of data elements

Returns:
    sparse or dense array of stacked data


x_features = self.compose_input_data(args)<EOL>try:<EOL><INDENT>predictions = self.infer_method(x_features)<EOL><DEDENT>except TypeError or ValueError:<EOL><INDENT>if issparse(x_features):<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>predictions = self.infer_method(x_features.todense())<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>".format(self.model_class))<EOL>predictions = self.infer_method(csr_matrix(x_features))<EOL><DEDENT><DEDENT>if isinstance(predictions, list):<EOL><INDENT>predictions_ = [[predictions[j][i][<NUM_LIT:1>] for j in range(len(predictions))] for i in range(x_features.shape[<NUM_LIT:0>])]<EOL>predictions = np.array(predictions_)<EOL><DEDENT>if self.ensure_list_output and len(predictions.shape) == <NUM_LIT:1>:<EOL><INDENT>predictions = predictions.reshape(-<NUM_LIT:1>, <NUM_LIT:1>)<EOL><DEDENT>if issparse(predictions):<EOL><INDENT>return predictions<EOL><DEDENT>else:<EOL><INDENT>return predictions.tolist()<EOL><DEDENT>

Infer on the given data according to given in the config infer method, \
    e.g. ``"predict", "predict_proba", "transform"``

Args:
    *args: list of inputs

Returns:
    predictions, e.g. list of labels, array of probability distribution, sparse array of vectorized samples


@overrides<EOL><INDENT>def save(self, epoch: Optional[int] = None) -> None:<DEDENT>

path = self.save_path<EOL>if epoch is not None:<EOL><INDENT>path = path.parent / self.epoch_save_path / str(epoch) / path.parts[-<NUM_LIT:1>]<EOL>path.resolve()<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>path.parent.mkdir(parents=True, exist_ok=True)<EOL>path = str(path)<EOL>log.info(f'<STR_LIT>')<EOL>with self.graph.as_default():<EOL><INDENT>saver = tf.train.Saver()<EOL>saver.save(self.sess, path)<EOL><DEDENT>


if hasattr(self, '<STR_LIT>'):<EOL><INDENT>for k in list(self.sess.graph.get_all_collection_keys()):<EOL><INDENT>self.sess.graph.clear_collection(k)<EOL><DEDENT><DEDENT>super().destroy()<EOL>

Delete model from memory

Returns:
    None


unique_indices, new_index_positions = tf.unique(indices)<EOL>summed_values = tf.unsorted_segment_sum(values,<EOL>new_index_positions,<EOL>tf.shape(unique_indices)[<NUM_LIT:0>])<EOL>return (summed_values, unique_indices)<EOL>

Sums `values` associated with any non-unique `indices`.
    Args:
      values: A `Tensor` with rank >= 1.
      indices: A one-dimensional integer `Tensor`, indexing into the first
      dimension of `values` (as in an IndexedSlices object).
    Returns:
      A tuple of (`summed_values`, `unique_indices`) where `unique_indices` is a
      de-duplicated version of `indices` and `summed_values` contains the sum of
      `values` slices associated with each unique index.


def module_fn():<EOL><INDENT>"""<STR_LIT>"""<EOL>_bos_id = <NUM_LIT><EOL>_eos_id = <NUM_LIT><EOL>_bow_id = <NUM_LIT><EOL>_eow_id = <NUM_LIT><EOL>_pad_id = <NUM_LIT><EOL>_max_word_length = <NUM_LIT:50><EOL>_parallel_iterations = <NUM_LIT:10><EOL>_max_batch_size = <NUM_LIT><EOL>id_dtype = tf.int32<EOL>id_nptype = np.int32<EOL>max_word_length = tf.constant(_max_word_length, dtype=id_dtype, name='<STR_LIT>')<EOL>version = tf.constant('<STR_LIT>', dtype=tf.string, name='<STR_LIT:version>')<EOL>def _make_bos_eos(c):<EOL><INDENT>r = np.zeros([_max_word_length], dtype=id_nptype)<EOL>r[:] = _pad_id<EOL>r[<NUM_LIT:0>] = _bow_id<EOL>r[<NUM_LIT:1>] = c<EOL>r[<NUM_LIT:2>] = _eow_id<EOL>return tf.constant(r, dtype=id_dtype)<EOL><DEDENT>bos_ids = _make_bos_eos(_bos_id)<EOL>eos_ids = _make_bos_eos(_eos_id)<EOL>def token2ids(token):<EOL><INDENT>with tf.name_scope("<STR_LIT>"):<EOL><INDENT>char_ids = tf.decode_raw(token, tf.uint8, name='<STR_LIT>')<EOL>char_ids = tf.cast(char_ids, tf.int32, name='<STR_LIT>')<EOL>char_ids = tf.strided_slice(char_ids, [<NUM_LIT:0>], [max_word_length - <NUM_LIT:2>],<EOL>[<NUM_LIT:1>], name='<STR_LIT>')<EOL>ids_num = tf.shape(char_ids)[<NUM_LIT:0>]<EOL>fill_ids_num = (_max_word_length - <NUM_LIT:2>) - ids_num<EOL>pads = tf.fill([fill_ids_num], _pad_id)<EOL>bow_token_eow_pads = tf.concat([[_bow_id], char_ids, [_eow_id], pads],<EOL><NUM_LIT:0>, name='<STR_LIT>')<EOL>return bow_token_eow_pads<EOL><DEDENT><DEDENT>def sentence_tagging_and_padding(sen_dim):<EOL><INDENT>with tf.name_scope("<STR_LIT>"):<EOL><INDENT>sen = sen_dim[<NUM_LIT:0>]<EOL>dim = sen_dim[<NUM_LIT:1>]<EOL>extra_dim = tf.shape(sen)[<NUM_LIT:0>] - dim<EOL>sen = tf.slice(sen, [<NUM_LIT:0>, <NUM_LIT:0>], [dim, max_word_length], name='<STR_LIT>')<EOL>bos_sen_eos = tf.concat([[bos_ids], sen, [eos_ids]], <NUM_LIT:0>, name='<STR_LIT>')<EOL>bos_sen_eos_plus_one = bos_sen_eos + <NUM_LIT:1><EOL>bos_sen_eos_pads = tf.pad(bos_sen_eos_plus_one, [[<NUM_LIT:0>, extra_dim], [<NUM_LIT:0>, <NUM_LIT:0>]],<EOL>"<STR_LIT>", name='<STR_LIT>')<EOL>return bos_sen_eos_pads<EOL><DEDENT><DEDENT>tokens = tf.placeholder(shape=(None, None), dtype=tf.string, name='<STR_LIT>')<EOL>sequence_len = tf.placeholder(shape=(None, ), dtype=tf.int32, name='<STR_LIT>')<EOL>tok_shape = tf.shape(tokens)<EOL>line_tokens = tf.reshape(tokens, shape=[-<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>tok_ids = tf.map_fn(<EOL>token2ids,<EOL>line_tokens,<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>tok_ids = tf.reshape(tok_ids, [tok_shape[<NUM_LIT:0>], tok_shape[<NUM_LIT:1>], -<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>sen_ids = tf.map_fn(<EOL>sentence_tagging_and_padding,<EOL>(tok_ids, sequence_len),<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>bilm = BidirectionalLanguageModel(options, str(weight_file),<EOL>max_batch_size=_max_batch_size)<EOL>embeddings_op = bilm(sen_ids)<EOL>elmo_output = weight_layers('<STR_LIT>', embeddings_op, l2_coef=<NUM_LIT:0.0>)<EOL>weighted_op = elmo_output['<STR_LIT>']<EOL>mean_op = elmo_output['<STR_LIT>']<EOL>word_emb = elmo_output['<STR_LIT>']<EOL>lstm_outputs1 = elmo_output['<STR_LIT>']<EOL>lstm_outputs2 = elmo_output['<STR_LIT>']<EOL>hub.add_signature("<STR_LIT>", {"<STR_LIT>": tokens, "<STR_LIT>": sequence_len},<EOL>{"<STR_LIT>": weighted_op,<EOL>"<STR_LIT:default>": mean_op,<EOL>"<STR_LIT>": word_emb,<EOL>"<STR_LIT>": lstm_outputs1,<EOL>"<STR_LIT>": lstm_outputs2,<EOL>"<STR_LIT:version>": version})<EOL>def_strings = tf.placeholder(shape=(None), dtype=tf.string)<EOL>def_tokens_sparse = tf.string_split(def_strings)<EOL>def_tokens_dense = tf.sparse_to_dense(sparse_indices=def_tokens_sparse.indices,<EOL>output_shape=def_tokens_sparse.dense_shape,<EOL>sparse_values=def_tokens_sparse.values,<EOL>default_value='<STR_LIT>'<EOL>)<EOL>def_mask = tf.not_equal(def_tokens_dense, '<STR_LIT>')<EOL>def_int_mask = tf.cast(def_mask, dtype=tf.int32)<EOL>def_sequence_len = tf.reduce_sum(def_int_mask, axis=-<NUM_LIT:1>)<EOL>def_tok_shape = tf.shape(def_tokens_dense)<EOL>def_line_tokens = tf.reshape(def_tokens_dense, shape=[-<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>def_tok_ids = tf.map_fn(<EOL>token2ids,<EOL>def_line_tokens,<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>def_tok_ids = tf.reshape(def_tok_ids, [def_tok_shape[<NUM_LIT:0>], def_tok_shape[<NUM_LIT:1>], -<NUM_LIT:1>], name='<STR_LIT>')<EOL>with tf.device('<STR_LIT>'):<EOL><INDENT>def_sen_ids = tf.map_fn(<EOL>sentence_tagging_and_padding,<EOL>(def_tok_ids, def_sequence_len),<EOL>dtype=tf.int32, back_prop=False, parallel_iterations=_parallel_iterations,<EOL>name='<STR_LIT>')<EOL><DEDENT>def_embeddings_op = bilm(def_sen_ids)<EOL>def_elmo_output = weight_layers('<STR_LIT>', def_embeddings_op, l2_coef=<NUM_LIT:0.0>, reuse=True)<EOL>def_weighted_op = def_elmo_output['<STR_LIT>']<EOL>def_mean_op = def_elmo_output['<STR_LIT>']<EOL>def_word_emb = def_elmo_output['<STR_LIT>']<EOL>def_lstm_outputs1 = def_elmo_output['<STR_LIT>']<EOL>def_lstm_outputs2 = def_elmo_output['<STR_LIT>']<EOL>hub.add_signature("<STR_LIT:default>", {"<STR_LIT>": def_strings},<EOL>{"<STR_LIT>": def_weighted_op,<EOL>"<STR_LIT:default>": def_mean_op,<EOL>"<STR_LIT>": def_word_emb,<EOL>"<STR_LIT>": def_lstm_outputs1,<EOL>"<STR_LIT>": def_lstm_outputs2,<EOL>"<STR_LIT:version>": version})<EOL><DEDENT>return hub.create_module_spec(module_fn)<EOL>

Makes a module spec.

    Args:
      options: LM hyperparameters.
      weight_file: location of the hdf5 file with LM weights.

    Returns:
      A module spec object used for constructing a TF-Hub module.


if not use_character_inputs:<EOL><INDENT>if embedding_weight_file is None:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>self._options = options<EOL>self._weight_file = weight_file<EOL>self._embedding_weight_file = embedding_weight_file<EOL>self._use_character_inputs = use_character_inputs<EOL>self._max_batch_size = max_batch_size<EOL>self._ops = {}<EOL>self._graphs = {}<EOL>

Creates the language model computational graph and loads weights

Two options for input type:
    (1) To use character inputs (paired with Batcher)
        pass use_character_inputs=True, and ids_placeholder
        of shape (None, None, max_characters_per_token)
        to __call__
    (2) To use token ids as input (paired with TokenBatcher),
        pass use_character_inputs=False and ids_placeholder
        of shape (None, None) to __call__.
        In this case, embedding_weight_file is also required input

options_file: location of the json formatted file with
              LM hyperparameters
weight_file: location of the hdf5 file with LM weights
use_character_inputs: if True, then use character ids as input,
    otherwise use token ids
max_batch_size: the maximum allowable batch size


def get_counts(self, docs: List[str], doc_ids: List[Any])-> Generator[Tuple[KeysView, ValuesView, List[int]], Any, None]:

logger.info("<STR_LIT>")<EOL>batch_ngrams = list(self.tokenizer(docs))<EOL>logger.info("<STR_LIT>")<EOL>doc_id = iter(doc_ids)<EOL>for ngrams in batch_ngrams:<EOL><INDENT>counts = Counter([hash_(gram, self.hash_size) for gram in ngrams])<EOL>hashes = counts.keys()<EOL>values = counts.values()<EOL>_id = self.doc_index[next(doc_id)]<EOL>if values:<EOL><INDENT>col_id = [_id] * len(values)<EOL><DEDENT>else:<EOL><INDENT>col_id = []<EOL><DEDENT>yield hashes, values, col_id<EOL><DEDENT>

Get term counts for a list of documents.

        Args:
            docs: a list of input documents
            doc_ids: a list of document ids corresponding to input documents

        Yields:
            a tuple of term hashes, count values and column ids

        Returns:
            None


if not isinstance(self.load_path, list):<EOL><INDENT>self.load_path = [self.load_path]<EOL><DEDENT>for i, path in enumerate(self.load_path):<EOL><INDENT>if isinstance(path, str):<EOL><INDENT>self.load_path[i] = pathlib.Path(path)<EOL><DEDENT><DEDENT>labels_by_words = defaultdict(set)<EOL>for infile in self.load_path:<EOL><INDENT>with infile.open("<STR_LIT:r>", encoding="<STR_LIT:utf8>") as fin:<EOL><INDENT>for line in fin:<EOL><INDENT>line = line.strip()<EOL>if line.count("<STR_LIT:\t>") != <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>word, labels = line.split("<STR_LIT:\t>")<EOL>labels_by_words[word].update(labels.split())<EOL><DEDENT><DEDENT><DEDENT>self._initialize(labels_by_words)<EOL>

Loads the dictionary from self.load_path


def __call__(self, data: List) -> np.ndarray:

<EOL>max_length = max(len(x) for x in data)<EOL>answer = np.zeros(shape=(len(data), max_length, self.dim), dtype=int)<EOL>for i, sent in enumerate(data):<EOL><INDENT>for j, word in enumerate(sent):<EOL><INDENT>answer[i, j][self._get_word_indexes(word)] = <NUM_LIT:1><EOL><DEDENT><DEDENT>return answer<EOL>

Transforms words to one-hot encoding according to the dictionary.

Args:
    data: the batch of words

Returns:
    a 3D array. answer[i][j][k] = 1 iff data[i][j] is the k-th word in the dictionary.


def __call__(self, input_doc_ids: List[List[Any]], input_doc_scores: List[List[float]]) ->Tuple[List[List], List[List]]:

batch_ids = []<EOL>batch_scores = []<EOL>for instance_ids, instance_scores in zip(input_doc_ids, input_doc_scores):<EOL><INDENT>instance_probas = []<EOL>for idx, score in zip(instance_ids, instance_scores):<EOL><INDENT>pop = self.pop_dict.get(idx, self.mean_pop)<EOL>features = [score, pop, score * pop]<EOL>prob = self.clf.predict_proba([features])<EOL>instance_probas.append(prob[<NUM_LIT:0>][<NUM_LIT:1>])<EOL><DEDENT>sort = sorted(enumerate(instance_probas), key=itemgetter(<NUM_LIT:1>), reverse=True)<EOL>sorted_probas = [item[<NUM_LIT:1>] for item in sort]<EOL>sorted_ids = [instance_ids[item[<NUM_LIT:0>]] for item in sort]<EOL>if self.active:<EOL><INDENT>sorted_ids = sorted_ids[:self.top_n]<EOL>sorted_probas = sorted_probas[:self.top_n]<EOL><DEDENT>batch_ids.append(sorted_ids)<EOL>batch_scores.append(sorted_probas)<EOL><DEDENT>return batch_ids, batch_scores<EOL>

Get tfidf scores and tfidf ids, re-rank them by applying logistic regression classifier,
        output pop ranker ids and pop ranker scores.

         Args:
            input_doc_ids: top input doc ids of tfidf ranker
            input_doc_scores: top input doc scores of tfidf ranker corresponding to doc ids

        Returns:
            top doc ids of pop ranker and their corresponding scores


if isinstance(key, str):<EOL><INDENT>return self.act2templ[key]<EOL><DEDENT>elif isinstance(key, Template):<EOL><INDENT>return self.templ2act[key]<EOL><DEDENT>

If key is an str, returns corresponding template.
        If key is a Template, return corresponding action.
        If does not exist, return None.


return hash(self.text)<EOL>

Override the default hash behavior (that returns the id)


@abstractmethod<EOL><INDENT>def get_features(self) -> np.ndarray:<DEDENT>

pass<EOL>

Returns:
    np.ndarray[float]: numpy array with calculates state features.


def cnn_model_max_and_aver_pool(self, kernel_sizes_cnn: List[int], filters_cnn: int, dense_size: int,<EOL>coef_reg_cnn: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>, dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>outputs = []<EOL>for i in range(len(kernel_sizes_cnn)):<EOL><INDENT>output_i = Conv1D(filters_cnn, kernel_size=kernel_sizes_cnn[i],<EOL>activation=None,<EOL>kernel_regularizer=l2(coef_reg_cnn),<EOL>padding='<STR_LIT>')(output)<EOL>output_i = BatchNormalization()(output_i)<EOL>output_i = Activation('<STR_LIT:relu>')(output_i)<EOL>output_i_0 = GlobalMaxPooling1D()(output_i)<EOL>output_i_1 = GlobalAveragePooling1D()(output_i)<EOL>output_i = Concatenate()([output_i_0, output_i_1])<EOL>outputs.append(output_i)<EOL><DEDENT>output = concatenate(outputs, axis=<NUM_LIT:1>)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled model of shallow-and-wide CNN where average pooling after convolutions is replaced with
concatenation of average and max poolings.

Args:
    kernel_sizes_cnn: list of kernel sizes of convolutions.
    filters_cnn: number of filters for convolutions.
    dense_size: number of units for dense layer.
    coef_reg_cnn: l2-regularization coefficient for convolutions. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate used after convolutions and between dense layers. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def pad_texts(self, sentences: List[List[np.ndarray]]) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:

pad = np.zeros(self.opt['<STR_LIT>'])<EOL>cutted_batch = [sen[:self.opt['<STR_LIT>']] for sen in sentences]<EOL>if self.opt["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>cutted_batch = [[pad] * (self.opt['<STR_LIT>'] - len(tokens)) + list(tokens) for tokens in cutted_batch]<EOL><DEDENT>elif self.opt["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>cutted_batch = [list(tokens) + [pad] * (self.opt['<STR_LIT>'] - len(tokens)) for tokens in cutted_batch]<EOL><DEDENT>else:<EOL><INDENT>raise ConfigError("<STR_LIT>".format(self.opt['<STR_LIT>']))<EOL><DEDENT>return np.asarray(cutted_batch)<EOL>

Cut and pad tokenized texts to self.opt["text_size"] tokens

Args:
    sentences: list of lists of tokens

Returns:
    array of embedded texts


def train_on_batch(self, texts: List[List[np.ndarray]], labels: list) -> Union[float, List[float]]:

features = self.check_input(texts)<EOL>metrics_values = self.model.train_on_batch(features, np.squeeze(np.array(labels)))<EOL>return metrics_values<EOL>

Train the model on the given batch

Args:
    texts: list of tokenized embedded text samples
    labels: list of labels

Returns:
    metrics values on the given batch


def check_input(self, texts: List[List[np.ndarray]]) -> np.ndarray:

if self.opt["<STR_LIT>"] is not None:<EOL><INDENT>features = self.pad_texts(texts)<EOL><DEDENT>else:<EOL><INDENT>if len(texts[<NUM_LIT:0>]):<EOL><INDENT>features = np.array(texts)<EOL><DEDENT>else:<EOL><INDENT>features = np.zeros((<NUM_LIT:1>, <NUM_LIT:1>, self.opt["<STR_LIT>"]))<EOL><DEDENT><DEDENT>return features<EOL>

Check and convert input to array of tokenized embedded samples

Args:
    texts: list of tokenized embedded text samples

Returns:
    array of tokenized embedded texts samples that are cut and padded


def __call__(self, data: Union[np.ndarray, List[List[float]], List[List[int]]],<EOL>*args, **kwargs) -> Union[List[List[str]], List[str]]:

if self.confident_threshold:<EOL><INDENT>return [list(np.where(np.array(d) > self.confident_threshold)[<NUM_LIT:0>])<EOL>for d in data]<EOL><DEDENT>elif self.max_proba:<EOL><INDENT>return [[np.argmax(d)] for d in data]<EOL><DEDENT>elif self.top_n:<EOL><INDENT>return [np.argsort(d)[::-<NUM_LIT:1>][:self.top_n] for d in data]<EOL><DEDENT>else:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Process probabilities to labels

Args:
    data: list of vectors with probability distribution
    *args:
    **kwargs:

Returns:
    list of labels (only label classification) or list of lists of labels (multi-label classification)


def train_on_batch(self, c_tokens: np.ndarray, c_chars: np.ndarray, q_tokens: np.ndarray, q_chars: np.ndarray,<EOL>y1s: Tuple[List[int], ...], y2s: Tuple[List[int], ...]) -> float:

<EOL>y1s = np.array([x[<NUM_LIT:0>] for x in y1s])<EOL>y2s = np.array([x[<NUM_LIT:0>] for x in y2s])<EOL>if self.noans_token:<EOL><INDENT>noans_mask = ((y1s != -<NUM_LIT:1>) * (y2s != -<NUM_LIT:1>))<EOL>y1s = (y1s + <NUM_LIT:1>) * noans_mask<EOL>y2s = (y2s + <NUM_LIT:1>) * noans_mask<EOL><DEDENT>feed_dict = self._build_feed_dict(c_tokens, c_chars, q_tokens, q_chars, y1s, y2s)<EOL>loss, _, lear_rate = self.sess.run([self.loss, self.train_op, self.lear_rate_ph],<EOL>feed_dict=feed_dict)<EOL>report = {'<STR_LIT>': loss, '<STR_LIT>': float(lear_rate), '<STR_LIT>': self.get_momentum()}<EOL>return report<EOL>

This method is called by trainer to make one training step on one batch.

Args:
    c_tokens: batch of tokenized contexts
    c_chars: batch of tokenized contexts, each token split on chars
    q_tokens: batch of tokenized questions
    q_chars: batch of tokenized questions, each token split on chars
    y1s: batch of ground truth answer start positions
    y2s: batch of ground truth answer end positions

Returns:
    value of loss function on batch


def attention(inputs, state, att_size, mask, scope="<STR_LIT>"):

with tf.variable_scope(scope):<EOL><INDENT>u = tf.concat([tf.tile(tf.expand_dims(state, axis=<NUM_LIT:1>), [<NUM_LIT:1>, tf.shape(inputs)[<NUM_LIT:1>], <NUM_LIT:1>]), inputs], axis=<NUM_LIT:2>)<EOL>logits = tf.layers.dense(tf.layers.dense(u, att_size, activation=tf.nn.tanh), <NUM_LIT:1>, use_bias=False)<EOL>logits = softmax_mask(tf.squeeze(logits, [<NUM_LIT:2>]), mask)<EOL>att_weights = tf.expand_dims(tf.nn.softmax(logits), axis=<NUM_LIT:2>)<EOL>res = tf.reduce_sum(att_weights * inputs, axis=<NUM_LIT:1>)<EOL>return res, logits<EOL><DEDENT>

Computes weighted sum of inputs conditioned on state


def dot_attention(inputs, memory, mask, att_size, keep_prob=<NUM_LIT:1.0>, scope="<STR_LIT>"):

with tf.variable_scope(scope):<EOL><INDENT>BS, IL, IH = tf.unstack(tf.shape(inputs))<EOL>BS, ML, MH = tf.unstack(tf.shape(memory))<EOL>d_inputs = tf.nn.dropout(inputs, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, IH])<EOL>d_memory = tf.nn.dropout(memory, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, MH])<EOL>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>inputs_att = tf.layers.dense(d_inputs, att_size, use_bias=False, activation=tf.nn.relu)<EOL>memory_att = tf.layers.dense(d_memory, att_size, use_bias=False, activation=tf.nn.relu)<EOL>logits = tf.matmul(inputs_att, tf.transpose(memory_att, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])) / (att_size ** <NUM_LIT:0.5>)<EOL>mask = tf.tile(tf.expand_dims(mask, axis=<NUM_LIT:1>), [<NUM_LIT:1>, IL, <NUM_LIT:1>])<EOL>att_weights = tf.nn.softmax(softmax_mask(logits, mask))<EOL>outputs = tf.matmul(att_weights, memory)<EOL>res = tf.concat([inputs, outputs], axis=<NUM_LIT:2>)<EOL><DEDENT>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>dim = res.get_shape().as_list()[-<NUM_LIT:1>]<EOL>d_res = tf.nn.dropout(res, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, IH + MH])<EOL>gate = tf.layers.dense(d_res, dim, use_bias=False, activation=tf.nn.sigmoid)<EOL>return res * gate<EOL><DEDENT><DEDENT>

Computes attention vector for each item in inputs:
       attention vector is a weighted sum of memory items.
       Dot product between input and memory vector is used as similarity measure.

       Gate mechanism is applied to attention vectors to produce output.

    Args:
        inputs: Tensor [batch_size x input_len x feature_size]
        memory: Tensor [batch_size x memory_len x feature_size]
        mask: inputs mask
        att_size: hidden size of attention
        keep_prob: dropout keep_prob
        scope:

    Returns:
        attention vectors [batch_size x input_len x (feature_size + feature_size)]


@overrides<EOL><INDENT>def __call__(self, batch: List[List[str]],<EOL>*args, **kwargs) -> Union[List[np.ndarray], np.ndarray]:<DEDENT>

if len(batch) > self.mini_batch_size:<EOL><INDENT>batch_gen = chunk_generator(batch, self.mini_batch_size)<EOL>elmo_output_values = []<EOL>for mini_batch in batch_gen:<EOL><INDENT>mini_batch_out = self._mini_batch_fit(mini_batch, *args, **kwargs)<EOL>elmo_output_values.extend(mini_batch_out)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>elmo_output_values = self._mini_batch_fit(batch, *args, **kwargs)<EOL><DEDENT>if self.pad_zero:<EOL><INDENT>elmo_output_values = zero_pad(elmo_output_values)<EOL><DEDENT>return elmo_output_values<EOL>

Embed sentences from a batch.

Args:
    batch: A list of tokenized text samples.

Returns:
    A batch of ELMo embeddings.


def _tags_encode(self, tokens: List[str], tags: List[str], mean: bool) -> Union[List[np.ndarray], np.ndarray]:

embedded_tokens = np.array(self.embedder([tokens]))[<NUM_LIT:0>, :, :]<EOL>tags_weights = np.array([self.tags_vocab.get(tag, <NUM_LIT:1.0>) for tag in tags])<EOL>detokenized_sample = self.tokenizer([tokens])[<NUM_LIT:0>]  <EOL>vectorized_sample = self.vectorizer([detokenized_sample])  <EOL>if self.vectorizer:<EOL><INDENT>weights = np.array([vectorized_sample[<NUM_LIT:0>, np.where(self.vocabulary == token)[<NUM_LIT:0>][<NUM_LIT:0>]]<EOL>if len(np.where(self.vocabulary == token)[<NUM_LIT:0>]) else <NUM_LIT:0.><EOL>for token in tokens])<EOL><DEDENT>else:<EOL><INDENT>weights = np.array([self.get_weight(max(self.counter_vocab.get(token, <NUM_LIT:0>), self.idf_base_count))<EOL>for token in tokens])<EOL><DEDENT>weights = np.multiply(weights, tags_weights)<EOL>if sum(weights) == <NUM_LIT:0>:<EOL><INDENT>weights = np.ones(len(tokens))<EOL><DEDENT>if mean is None:<EOL><INDENT>mean = self.mean<EOL><DEDENT>if mean:<EOL><INDENT>embedded_tokens = np.average(embedded_tokens, weights=weights, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>embedded_tokens = np.array([weights[i] * embedded_tokens[i] for i in range(len(tokens))])<EOL><DEDENT>return embedded_tokens<EOL>

Embed one text sample

Args:
    tokens: tokenized text sample
    tags: tokenized tags sample
    mean: whether to return mean token embedding (does not depend on self.mean)

Returns:
    list of embedded tokens or array of mean values


@overrides<EOL><INDENT>def __call__(self, batch: List[List[str]], tags_batch: Optional[List[List[str]]] = None, mean: bool = None,<EOL>*args, **kwargs) -> List[Union[list, np.ndarray]]:<DEDENT>

if self.tags_vocab:<EOL><INDENT>if tags_batch is None:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>batch = [self._tags_encode(sample, tags_sample, mean=mean) for sample, tags_sample in zip(batch, tags_batch)]<EOL><DEDENT>else:<EOL><INDENT>if tags_batch:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>batch = [self._encode(sample, mean=mean) for sample in batch]<EOL><DEDENT>if self.pad_zero:<EOL><INDENT>batch = zero_pad(batch)<EOL><DEDENT>return batch<EOL>

Infer on the given data

Args:
    batch: tokenized text samples
    tags_batch: optional batch of corresponding tags
    mean: whether to return mean token embedding (does not depend on self.mean)
    *args: additional arguments
    **kwargs: additional arguments

Returns:


@staticmethod<EOL><INDENT>def space_detokenizer(batch: List[List[str]]) -> List[str]:<DEDENT>

return ["<STR_LIT:U+0020>".join(tokens) for tokens in batch]<EOL>

Detokenizer by default. Linking tokens by space symbol

Args:
    batch: batch of tokenized texts

Returns:
    batch of detokenized texts


def _encode(self, tokens: List[str], mean: bool) -> Union[List[np.ndarray], np.ndarray]:

if self.vectorizer:<EOL><INDENT>detokenized_sample = self.tokenizer([tokens])[<NUM_LIT:0>]  <EOL>vectorized_sample = self.vectorizer([detokenized_sample])  <EOL>weights = np.array([vectorized_sample[<NUM_LIT:0>, np.where(self.vocabulary == token)[<NUM_LIT:0>][<NUM_LIT:0>]]<EOL>if len(np.where(self.vocabulary == token)[<NUM_LIT:0>]) else <NUM_LIT:0.><EOL>for token in tokens])<EOL><DEDENT>else:<EOL><INDENT>weights = np.array([self.get_weight(max(self.counter_vocab.get(token, <NUM_LIT:0>), self.idf_base_count))<EOL>for token in tokens])<EOL><DEDENT>if sum(weights) == <NUM_LIT:0>:<EOL><INDENT>weights = np.ones(len(tokens))<EOL><DEDENT>embedded_tokens = np.array(self.embedder([tokens]))[<NUM_LIT:0>, :, :]<EOL>if mean is None:<EOL><INDENT>mean = self.mean<EOL><DEDENT>if mean:<EOL><INDENT>embedded_tokens = np.average(embedded_tokens, weights=weights, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>embedded_tokens = np.array([weights[i] * embedded_tokens[i] for i in range(len(tokens))])<EOL><DEDENT>return embedded_tokens<EOL>

Embed one text sample

Args:
    tokens: tokenized text sample
    mean: whether to return mean token embedding (does not depend on self.mean)

Returns:
    list of embedded tokens or array of mean values


log.info(f"<STR_LIT>")<EOL>if not self.load_path.exists():<EOL><INDENT>log.warning(f'<STR_LIT>')<EOL>return<EOL><DEDENT>self.model = KeyedVectors.load_word2vec_format(str(self.load_path))<EOL>self.dim = self.model.vector_size<EOL>


def __call__(self, batch: List[str]) -> List[List[str]]:

if isinstance(batch, (list, tuple)):<EOL><INDENT>return [sample.split() for sample in batch]<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>

Tokenize given batch

Args:
    batch: list of texts to tokenize

Returns:
    tokenized batch


def __call__(self, batch: List[Union[str, List[str]]]) -> List[Union[List[str], str]]:

if isinstance(batch[<NUM_LIT:0>], str):<EOL><INDENT>return [self.tokenizer.tokenize(line, escape=self.escape) for line in batch]<EOL><DEDENT>else:<EOL><INDENT>return [self.detokenizer.detokenize(line, return_str=True, unescape=self.escape)<EOL>for line in batch]<EOL><DEDENT>

Tokenize given batch of strings or detokenize given batch of lists of tokens

        Args:
            batch: list of text samples or list of lists of tokens

        Returns:
            list of lists of tokens or list of text samples


text = '<STR_LIT:U+0020>'.join(tokens)<EOL>step0 = text.replace('<STR_LIT>', '<STR_LIT>')<EOL>step1 = step0.replace("<STR_LIT>", '<STR_LIT:">').replace("<STR_LIT>", '<STR_LIT:">')<EOL>step2 = step1.replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>")<EOL>step3 = re.sub(r'<STR_LIT>', r"<STR_LIT>", step2)<EOL>step4 = re.sub(r'<STR_LIT>', r"<STR_LIT>", step3)<EOL>step5 = step4.replace("<STR_LIT>", "<STR_LIT:'>").replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>")<EOL>step6 = step5.replace("<STR_LIT>", "<STR_LIT>")<EOL>return step6.strip()<EOL>

Detokenizing a text undoes the tokenizing operation, restores
punctuation and spaces to the places that people expect them to be.
Ideally, `detokenize(tokenize(text))` should be identical to `text`,
except for line breaks.


args = parser.parse_args()<EOL>run_ms_bot_framework_server(agent_generator=make_agent,<EOL>app_id=args.ms_id,<EOL>app_secret=args.ms_secret,<EOL>stateful=True)<EOL>

Parse parameters and run ms bot framework


def show_details(item_data: Dict[Any, Any]) -> str:

txt = "<STR_LIT>"<EOL>for key, value in item_data.items():<EOL><INDENT>txt += "<STR_LIT>" + str(key) + "<STR_LIT>" + '<STR_LIT>' + str(value) + "<STR_LIT>"<EOL><DEDENT>return txt<EOL>

Format catalog item output

    Parameters:
        item_data: item's attributes values

    Returns:
        [rich_message]: list of formatted rich message


self.control_json['<STR_LIT:content>'] = self.content<EOL>return self.control_json<EOL>

Returns json compatible state of the PlainText instance.

        Returns:
            control_json: Json representation of PlainText state.


self._load()<EOL>return self._chainer<EOL>

Return a :class:`~deeppavlov.core.common.chainer.Chainer` built from ``self.chainer_config`` for inference


def add_control(self, control: RichControl):

self.controls.append(control)<EOL>

Adds RichControl instance to RichMessage.

        Args:
            control: RichControl instance.


return None<EOL>

Returns Telegram compatible state of the control instance
        including its nested controls.

        Returns:
            control: Telegram representation of control state.


alexa_controls = [control.alexa() for control in self.controls]<EOL>return alexa_controls<EOL>

Returns list of Amazon Alexa compatible states of the RichMessage
        instance nested controls.

        Returns:
            alexa_controls: Amazon Alexa representation of RichMessage instance nested
                controls.


@abstractmethod<EOL><INDENT>def json(self) -> Union[list, dict]:<DEDENT>

pass<EOL>

Returns json compatible state of the control instance including
        its nested controls.

        Returns:
            control: Json representation of control state.


json_controls = [control.json() for control in self.controls]<EOL>return json_controls<EOL>

Returns list of json compatible states of the RichMessage instance
        nested controls.

        Returns:
            json_controls: Json representation of RichMessage instance
                nested controls.


def _log(self, utterance: Any, direction: str, dialog_id: Optional[Hashable]=None):

if isinstance(utterance, str):<EOL><INDENT>pass<EOL><DEDENT>elif isinstance(utterance, RichMessage):<EOL><INDENT>utterance = utterance.json()<EOL><DEDENT>elif isinstance(utterance, (list, dict)):<EOL><INDENT>utterance = jsonify_data(utterance)<EOL><DEDENT>else:<EOL><INDENT>utterance = str(utterance)<EOL><DEDENT>dialog_id = str(dialog_id) if not isinstance(dialog_id, str) else dialog_id<EOL>if self.log_file.tell() >= self.log_max_size * <NUM_LIT>:<EOL><INDENT>self.log_file.close()<EOL>self.log_file = self._get_log_file()<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>log_msg = {}<EOL>log_msg['<STR_LIT>'] = self._get_timestamp_utc_str()<EOL>log_msg['<STR_LIT>'] = dialog_id<EOL>log_msg['<STR_LIT>'] = direction<EOL>log_msg['<STR_LIT:message>'] = utterance<EOL>log_str = json.dumps(log_msg, ensure_ascii=self.config['<STR_LIT>'])<EOL>self.log_file.write(f'<STR_LIT>')<EOL><DEDENT>except IOError:<EOL><INDENT>log.error('<STR_LIT>')<EOL><DEDENT><DEDENT>

Logs single dialog utterance to current dialog log file.

        Args:
            utterance: Dialog utterance.
            direction: 'in' or 'out' utterance direction.
            dialog_id: Dialog ID.


def log_in(self, utterance: Any, dialog_id: Optional[Hashable] = None) -> None:

if self.enabled:<EOL><INDENT>self._log(utterance, '<STR_LIT>', dialog_id)<EOL><DEDENT>

Wraps _log method for all input utterances.
        Args:
            utterance: Dialog utterance.
            dialog_id: Dialog ID.


@abstractmethod<EOL><INDENT>def _call(self, utterances_batch: list, utterances_ids: Optional[list] = None) -> list:<DEDENT>

pass<EOL>

Processes batch of utterances and returns corresponding responses batch.

        Each call of Agent processes incoming utterances and returns response
        for each utterance Batch of dialog IDs can be provided, in other case
        utterances indexes in incoming batch are used as dialog IDs.

        Args:
            utterances_batch: Batch of incoming utterances.
            utterances_ids: Batch of dialog IDs corresponding to incoming utterances.

        Returns:
            responses: A batch of responses corresponding to the
                utterance batch received by agent.


log.info('<STR_LIT>')<EOL>variables = tf.trainable_variables()<EOL>blocks = defaultdict(int)<EOL>for var in variables:<EOL><INDENT>block_name = var.name.split('<STR_LIT:/>')[<NUM_LIT:0>]<EOL>number_of_parameters = np.prod(var.get_shape().as_list())<EOL>blocks[block_name] += number_of_parameters<EOL><DEDENT>for block_name, cnt in blocks.items():<EOL><INDENT>log.info("<STR_LIT>".format(block_name, cnt))<EOL><DEDENT>total_num_parameters = np.sum(list(blocks.values()))<EOL>log.info('<STR_LIT>'.format(total_num_parameters))<EOL>

Print number of *trainable* parameters in the network


if optimizer_scope_name is None:<EOL><INDENT>opt_scope = tf.variable_scope('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>opt_scope = tf.variable_scope(optimizer_scope_name)<EOL><DEDENT>with opt_scope:<EOL><INDENT>if learnable_scopes is None:<EOL><INDENT>variables_to_train = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)<EOL><DEDENT>else:<EOL><INDENT>variables_to_train = []<EOL>for scope_name in learnable_scopes:<EOL><INDENT>variables_to_train.extend(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope_name))<EOL><DEDENT><DEDENT>if optimizer is None:<EOL><INDENT>optimizer = tf.train.AdamOptimizer<EOL><DEDENT>extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)<EOL>with tf.control_dependencies(extra_update_ops):<EOL><INDENT>def clip_if_not_none(grad):<EOL><INDENT>if grad is not None:<EOL><INDENT>return tf.clip_by_norm(grad, clip_norm)<EOL><DEDENT><DEDENT>opt = optimizer(learning_rate, **kwargs)<EOL>grads_and_vars = opt.compute_gradients(loss, var_list=variables_to_train)<EOL>if clip_norm is not None:<EOL><INDENT>grads_and_vars = [(clip_if_not_none(grad), var)<EOL>for grad, var in grads_and_vars]<EOL><DEDENT>train_op = opt.apply_gradients(grads_and_vars)<EOL><DEDENT><DEDENT>return train_op<EOL>

Get train operation for given loss

Args:
    loss: loss, tf tensor or scalar
    learning_rate: scalar or placeholder.
    clip_norm: clip gradients norm by clip_norm.
    learnable_scopes: which scopes are trainable (None for all).
    optimizer: instance of tf.train.Optimizer, default Adam.
    **kwargs: parameters passed to tf.train.Optimizer object
       (scalars or placeholders).

Returns:
    train_op


<EOL>if self.load_path.exists():<EOL><INDENT>path = str(self.load_path.resolve())<EOL>log.info('<STR_LIT>'.format(path))<EOL>self._net.load(path)<EOL><DEDENT>

Checks existence of the model file, loads the model if the file exists


pass<EOL>

Return instance of keras optimizer

Args:
    None


def __call__(self, *x_batch, **kwargs) -> Union[List, np.ndarray]:

with self.graph.as_default():<EOL><INDENT>K.set_session(self.sess)<EOL>return self._net.predict_on_batch(x_batch, **kwargs)<EOL><DEDENT>

Predicts answers on batch elements.

Args:
    instance: a batch to predict answers on


def process_event(self, event_name: str, data: dict):

if (isinstance(self.opt.get("<STR_LIT>", None), float) and<EOL>isinstance(self.opt.get("<STR_LIT>", None), float)):<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>if event_name == '<STR_LIT>':<EOL><INDENT>if (self.get_learning_rate_variable() is not None) and ('<STR_LIT>' not in data):<EOL><INDENT>data['<STR_LIT>'] = float(K.get_value(self.get_learning_rate_variable()))<EOL><DEDENT>if (self.get_momentum_variable() is not None) and ('<STR_LIT>' not in data):<EOL><INDENT>data['<STR_LIT>'] = float(K.get_value(self.get_momentum_variable()))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>super().process_event(event_name, data)<EOL><DEDENT><DEDENT>

Process event after epoch
Args:
    event_name: whether event is send after epoch or batch.
            Set of values: ``"after_epoch", "after_batch"``
    data: event data (dictionary)

Returns:
    None


path = str(self.save_path.absolute())<EOL>log.info('<STR_LIT>'.format(path))<EOL>self._net.save(path)<EOL>

Saves model to the save_path, provided in config. The directory is
        already created by super().__init__, which is called in __init__ of this class


return self._lr_var<EOL>

Return current learning rate variable

Returns:
    learning rate variable


def __init__(self,<EOL>learning_rate: Union[None, float, Tuple[float, float]] = None,<EOL>learning_rate_decay: Union[DType, Tuple[DType, float]] = DecayType.NO,<EOL>learning_rate_decay_epochs: int = <NUM_LIT:0>,<EOL>learning_rate_decay_batches: int = <NUM_LIT:0>,<EOL>learning_rate_drop_div: float = <NUM_LIT>,<EOL>learning_rate_drop_patience: Optional[int] = None,<EOL>momentum: Union[None, float, Tuple[float, float]] = None,<EOL>momentum_decay: Union[DType, Tuple[DType, float]] = DecayType.NO,<EOL>momentum_decay_epochs: int = <NUM_LIT:0>,<EOL>momentum_decay_batches: int = <NUM_LIT:0>,<EOL>fit_batch_size: Union[None, int, str] = None,<EOL>fit_learning_rate: Tuple[float, float] = (<NUM_LIT>, <NUM_LIT:100>),<EOL>fit_learning_rate_div: float = <NUM_LIT>,<EOL>fit_beta: float = <NUM_LIT>,<EOL>fit_min_batches: int = <NUM_LIT:10>,<EOL>fit_max_batches: Optional[int] = None,<EOL>*args, **kwargs) -> None:

if learning_rate_decay_epochs and learning_rate_decay_batches:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>if momentum_decay_epochs and momentum_decay_batches:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>start_val, end_val = learning_rate, None<EOL>if isinstance(learning_rate, (tuple, list)):<EOL><INDENT>start_val, end_val = learning_rate<EOL><DEDENT>dec_type, extra = learning_rate_decay, None<EOL>if isinstance(learning_rate_decay, (tuple, list)):<EOL><INDENT>dec_type, extra = learning_rate_decay<EOL><DEDENT>self._lr = start_val<EOL>num_it, self._lr_update_on_batch = learning_rate_decay_epochs, False<EOL>if learning_rate_decay_batches > <NUM_LIT:0>:<EOL><INDENT>num_it, self._lr_update_on_batch = learning_rate_decay_batches, True<EOL><DEDENT>self._lr_schedule = DecayScheduler(start_val=start_val, end_val=end_val,<EOL>num_it=num_it, dec_type=dec_type, extra=extra)<EOL>self._lr_var = self._init_learning_rate_variable()<EOL>start_val, end_val = momentum, None<EOL>if isinstance(momentum, (tuple, list)):<EOL><INDENT>start_val, end_val = momentum<EOL><DEDENT>dec_type, extra = momentum_decay, None<EOL>if isinstance(momentum_decay, (tuple, list)):<EOL><INDENT>dec_type, extra = momentum_decay<EOL><DEDENT>self._mom = start_val<EOL>num_it, self._mom_update_on_batch = momentum_decay_epochs, False<EOL>self._mom_update_on_batch = momentum_decay_batches > <NUM_LIT:0><EOL>num_it = momentum_decay_epochs if self._mom_update_on_batch else momentum_decay_batches<EOL>self._mom_schedule = DecayScheduler(start_val=start_val, end_val=end_val,<EOL>num_it=num_it, dec_type=dec_type,<EOL>extra=extra)<EOL>self._mom_var = self._init_momentum_variable()<EOL>self._learning_rate_drop_patience = learning_rate_drop_patience<EOL>self._learning_rate_drop_div = learning_rate_drop_div<EOL>self._learning_rate_cur_impatience = <NUM_LIT:0.><EOL>self._learning_rate_last_impatience = <NUM_LIT:0.><EOL>self._learning_rate_cur_div = <NUM_LIT:1.><EOL>self._fit_batch_size = fit_batch_size<EOL>self._fit_learning_rate = fit_learning_rate<EOL>self._fit_learning_rate_div = fit_learning_rate_div<EOL>self._fit_beta = fit_beta<EOL>self._fit_min_batches = fit_min_batches<EOL>self._fit_max_batches = fit_max_batches<EOL>


return self._mom_var<EOL>

Return current momentum variable

Returns:
    momentum variable


@staticmethod<EOL><INDENT>def _get_best(values: List[float], losses: List[float],<EOL>max_loss_div: float = <NUM_LIT>, min_val_div: float = <NUM_LIT>) -> float:<DEDENT>

assert len(values) == len(losses), "<STR_LIT>"<EOL>min_ind = np.argmin(losses)<EOL>for i in range(min_ind - <NUM_LIT:1>, <NUM_LIT:0>, -<NUM_LIT:1>):<EOL><INDENT>if (losses[i] * max_loss_div > losses[min_ind]) or(values[i] * min_val_div < values[min_ind]):<EOL><INDENT>return values[i + <NUM_LIT:1>]<EOL><DEDENT><DEDENT>return values[min_ind] / min_val_div<EOL>

Find the best value according to given losses

Args:
    values: list of considered values
    losses: list of obtained loss values corresponding to `values`
    max_loss_div: maximal divergence of loss to be considered significant
    min_val_div: minimum divergence of loss to be considered significant

Returns:
    best value divided by `min_val_div`


l2_reg = tf.nn.l2_loss if add_l2_losses else None<EOL>for n_layer, n_hidden in enumerate(n_hidden_list):<EOL><INDENT>if use_dilation:<EOL><INDENT>dilation_rate = <NUM_LIT:2> ** n_layer<EOL><DEDENT>else:<EOL><INDENT>dilation_rate = <NUM_LIT:1><EOL><DEDENT>units = tf.layers.conv1d(units,<EOL>n_hidden,<EOL>filter_width,<EOL>padding='<STR_LIT>',<EOL>dilation_rate=dilation_rate,<EOL>kernel_initializer=INITIALIZER(),<EOL>kernel_regularizer=l2_reg)<EOL>if use_batch_norm:<EOL><INDENT>assert training_ph is not None<EOL>units = tf.layers.batch_normalization(units, training=training_ph)<EOL><DEDENT>units = tf.nn.relu(units)<EOL><DEDENT>return units<EOL>

Number of convolutional layers stacked on top of each other

    Args:
        units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
        n_hidden_list: list with number of hidden units at the ouput of each layer
        filter_width: width of the kernel in tokens
        use_batch_norm: whether to use batch normalization between layers
        use_dilation: use power of 2 dilation scheme [1, 2, 4, 8 .. ] for layers 1, 2, 3, 4 ...
        training_ph: boolean placeholder determining whether is training phase now or not.
            It is used only for batch normalization to determine whether to use
            current batch average (std) or memory stored average (std)
        add_l2_losses: whether to add l2 losses on network kernels to
                tf.GraphKeys.REGULARIZATION_LOSSES or not

    Returns:
        units: tensor at the output of the last convolutional layer


for n, n_hidden in enumerate(n_hidden_list):<EOL><INDENT>with tf.variable_scope(name + '<STR_LIT:_>' + str(n)):<EOL><INDENT>if cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.GRUCell(n_hidden)<EOL>backward_cell = tf.nn.rnn_cell.GRUCell(n_hidden)<EOL><DEDENT>elif cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes)<EOL>backward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes)<EOL><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>(rnn_output_fw, rnn_output_bw), (fw, bw) =tf.nn.bidirectional_dynamic_rnn(forward_cell,<EOL>backward_cell,<EOL>units,<EOL>dtype=tf.float32,<EOL>sequence_length=seq_lengths)<EOL>units = tf.concat([rnn_output_fw, rnn_output_bw], axis=<NUM_LIT:2>)<EOL>if cell_type == '<STR_LIT>':<EOL><INDENT>last_units = tf.concat([fw, bw], axis=<NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>(c_fw, h_fw), (c_bw, h_bw) = fw, bw<EOL>c = tf.concat([c_fw, c_bw], axis=<NUM_LIT:1>)<EOL>h = tf.concat([h_fw, h_bw], axis=<NUM_LIT:1>)<EOL>last_units = (h, c)<EOL><DEDENT><DEDENT><DEDENT>return units, last_units<EOL>

Stackted recurrent neural networks GRU or LSTM

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden_list: list with number of hidden units at the ouput of each layer
            seq_lengths: length of sequences for different length sequences in batch
                can be None for maximum length as a length for every sample in the batch
            cell_type: 'lstm' or 'gru'
            use_peepholes: whether to use peephole connections (only 'lstm' case affected)
            name: what variable_scope to use for the network parameters
        Returns:
            units: tensor at the output of the last recurrent layer
                with dimensionality [None, n_tokens, n_hidden_list[-1]]
            last_units: tensor of last hidden states for GRU and tuple
                of last hidden stated and last cell states for LSTM
                dimensionality of cell states and hidden states are
                similar and equal to [B x 2 * H], where B - batch
                size and H is number of hidden units


def multiplicative_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = units.get_shape().as_list()[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>queries = tf.layers.dense(expand_tile(units, <NUM_LIT:1>), n_hidden, kernel_initializer=INITIALIZER())<EOL>keys = tf.layers.dense(expand_tile(units, <NUM_LIT:2>), n_hidden, kernel_initializer=INITIALIZER())<EOL>scores = tf.reduce_sum(queries * keys, axis=<NUM_LIT:3>, keep_dims=True)<EOL>attention = tf.nn.softmax(scores, dim=<NUM_LIT:2>)<EOL>attended_units = tf.reduce_sum(attention * expand_tile(units, <NUM_LIT:1>), axis=<NUM_LIT:2>)<EOL>output = tf.layers.dense(attended_units, n_output_features, activation, kernel_initializer=INITIALIZER())<EOL>return output<EOL>

Computes multiplicative self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <W_1 h_i,  W_2 h_j>,  W_1 and W_2 are learnable matrices
        with dimensionality [n_hidden, n_input_features], where <a, b> stands for a and b
        dot product

    Args:
        units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
        n_hidden: number of units in hidden representation of similarity measure
        n_output_features: number of features in output dense layer
        activation: activation at the output

    Returns:
        output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


with tf.variable_scope(name + '<STR_LIT:_>' + cell_type.upper()):<EOL><INDENT>if cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.GRUCell(n_hidden, kernel_initializer=INITIALIZER())<EOL>backward_cell = tf.nn.rnn_cell.GRUCell(n_hidden, kernel_initializer=INITIALIZER())<EOL>if trainable_initial_states:<EOL><INDENT>initial_state_fw = tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>initial_state_bw = tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>))<EOL><DEDENT>else:<EOL><INDENT>initial_state_fw = initial_state_bw = None<EOL><DEDENT><DEDENT>elif cell_type == '<STR_LIT>':<EOL><INDENT>forward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes, initializer=INITIALIZER())<EOL>backward_cell = tf.nn.rnn_cell.LSTMCell(n_hidden, use_peepholes=use_peepholes, initializer=INITIALIZER())<EOL>if trainable_initial_states:<EOL><INDENT>initial_state_fw = tf.nn.rnn_cell.LSTMStateTuple(<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)),<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)))<EOL>initial_state_bw = tf.nn.rnn_cell.LSTMStateTuple(<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)),<EOL>tf.tile(tf.get_variable('<STR_LIT>', [<NUM_LIT:1>, n_hidden]), (tf.shape(units)[<NUM_LIT:0>], <NUM_LIT:1>)))<EOL><DEDENT>else:<EOL><INDENT>initial_state_fw = initial_state_bw = None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>(rnn_output_fw, rnn_output_bw), (fw, bw) =tf.nn.bidirectional_dynamic_rnn(forward_cell,<EOL>backward_cell,<EOL>units,<EOL>dtype=tf.float32,<EOL>sequence_length=seq_lengths,<EOL>initial_state_fw=initial_state_fw,<EOL>initial_state_bw=initial_state_bw)<EOL><DEDENT>kernels = [var for var in forward_cell.trainable_variables +<EOL>backward_cell.trainable_variables if '<STR_LIT>' in var.name]<EOL>for kernel in kernels:<EOL><INDENT>tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, tf.nn.l2_loss(kernel))<EOL><DEDENT>return (rnn_output_fw, rnn_output_bw), (fw, bw)<EOL>

Bi directional recurrent neural network. GRU or LSTM

        Args:
            units: a tensorflow tensor with dimensionality [None, n_tokens, n_features]
            n_hidden: list with number of hidden units at the ouput of each layer
            seq_lengths: length of sequences for different length sequences in batch
                can be None for maximum length as a length for every sample in the batch
            cell_type: 'lstm' or 'gru'
            trainable_initial_states: whether to create a special trainable variable
                to initialize the hidden states of the network or use just zeros
            use_peepholes: whether to use peephole connections (only 'lstm' case affected)
            name: what variable_scope to use for the network parameters

        Returns:
            units: tensor at the output of the last recurrent layer
                with dimensionality [None, n_tokens, n_hidden_list[-1]]
            last_units: tensor of last hidden states for GRU and tuple
                of last hidden stated and last cell states for LSTM
                dimensionality of cell states and hidden states are
                similar and equal to [B x 2 * H], where B - batch
                size and H is number of hidden units