def weight_layers(name, bilm_ops, l2_coef=None,<EOL>use_top_only=False, do_layer_norm=False, reuse=False):

def _l2_regularizer(weights):<EOL><INDENT>if l2_coef is not None:<EOL><INDENT>return l2_coef * tf.reduce_sum(tf.square(weights))<EOL><DEDENT>else:<EOL><INDENT>return <NUM_LIT:0.0><EOL><DEDENT><DEDENT>lm_embeddings = bilm_ops['<STR_LIT>']<EOL>mask = bilm_ops['<STR_LIT>']<EOL>n_lm_layers = int(lm_embeddings.get_shape()[<NUM_LIT:1>])<EOL>lm_dim = int(lm_embeddings.get_shape()[<NUM_LIT:3>])<EOL>with tf.control_dependencies([lm_embeddings, mask]):<EOL><INDENT>mask_float = tf.cast(mask, '<STR_LIT>')<EOL>broadcast_mask = tf.expand_dims(mask_float, axis=-<NUM_LIT:1>)<EOL>def _do_ln(x):<EOL><INDENT>x_masked = x * broadcast_mask<EOL>N = tf.reduce_sum(mask_float) * lm_dim<EOL>mean = tf.reduce_sum(x_masked) / N<EOL>variance = tf.reduce_sum(((x_masked - mean) * broadcast_mask)**<NUM_LIT:2>) / N<EOL>return tf.nn.batch_normalization(<EOL>x, mean, variance, None, None, <NUM_LIT><EOL>)<EOL><DEDENT>if use_top_only:<EOL><INDENT>layers = tf.split(lm_embeddings, n_lm_layers, axis=<NUM_LIT:1>)<EOL>sum_pieces = tf.squeeze(layers[-<NUM_LIT:1>], squeeze_dims=<NUM_LIT:1>)<EOL>reg = <NUM_LIT:0.0><EOL><DEDENT>else:<EOL><INDENT>with tf.variable_scope("<STR_LIT>", reuse=reuse):<EOL><INDENT>W = tf.get_variable(<EOL>'<STR_LIT>'.format(name),<EOL>shape=(n_lm_layers, ),<EOL>initializer=tf.zeros_initializer,<EOL>regularizer=_l2_regularizer,<EOL>trainable=True,<EOL>)<EOL><DEDENT>normed_weights = tf.split(<EOL>tf.nn.softmax(W + <NUM_LIT:1.0> / n_lm_layers), n_lm_layers<EOL>)<EOL>layers = tf.split(lm_embeddings, n_lm_layers, axis=<NUM_LIT:1>)<EOL>pieces = []<EOL>for w, t in zip(normed_weights, layers):<EOL><INDENT>if do_layer_norm:<EOL><INDENT>pieces.append(w * _do_ln(tf.squeeze(t, squeeze_dims=<NUM_LIT:1>)))<EOL><DEDENT>else:<EOL><INDENT>pieces.append(w * tf.squeeze(t, squeeze_dims=<NUM_LIT:1>))<EOL><DEDENT><DEDENT>sum_pieces = tf.add_n(pieces)<EOL>reg = [<EOL>r for r in tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)<EOL>if r.name.find('<STR_LIT>'.format(name)) >= <NUM_LIT:0><EOL>]<EOL>if len(reg) != <NUM_LIT:1>:<EOL><INDENT>raise ValueError<EOL><DEDENT><DEDENT>with tf.variable_scope("<STR_LIT>", reuse=reuse):<EOL><INDENT>gamma = tf.get_variable(<EOL>'<STR_LIT>'.format(name),<EOL>shape=(<NUM_LIT:1>, ),<EOL>initializer=tf.ones_initializer,<EOL>regularizer=None,<EOL>trainable=True,<EOL>)<EOL><DEDENT>weighted_lm_layers = sum_pieces * gamma<EOL>weighted_lm_layers_masked = sum_pieces * broadcast_mask<EOL>weighted_lm_layers_sum = tf.reduce_sum(weighted_lm_layers_masked, <NUM_LIT:1>)<EOL>mask_sum = tf.reduce_sum(mask_float, <NUM_LIT:1>)<EOL>mask_sum = tf.maximum(mask_sum, [<NUM_LIT:1>])<EOL>weighted_lm_layers_mean = weighted_lm_layers_sum / tf.expand_dims(mask_sum, - <NUM_LIT:1>)<EOL>word_emb_2n = tf.squeeze(layers[<NUM_LIT:0>], [<NUM_LIT:1>])<EOL>word_emb_1n = tf.slice(word_emb_2n, [<NUM_LIT:0>, <NUM_LIT:0>, <NUM_LIT:0>], [-<NUM_LIT:1>, -<NUM_LIT:1>, lm_dim // <NUM_LIT:2>])  <EOL>lstm_outputs1 = tf.squeeze(layers[<NUM_LIT:1>], [<NUM_LIT:1>])<EOL>lstm_outputs2 = tf.squeeze(layers[<NUM_LIT:2>], [<NUM_LIT:1>])<EOL>ret = {'<STR_LIT>': weighted_lm_layers,<EOL>'<STR_LIT>': weighted_lm_layers_mean,<EOL>'<STR_LIT>': reg,<EOL>'<STR_LIT>': word_emb_1n,<EOL>'<STR_LIT>': lstm_outputs1,<EOL>'<STR_LIT>': lstm_outputs2, }<EOL><DEDENT>return ret<EOL>

Weight the layers of a biLM with trainable scalar weights to
compute ELMo representations.

For each output layer, this returns two ops.  The first computes
    a layer specific weighted average of the biLM layers, and
    the second the l2 regularizer loss term.
The regularization terms are also add to tf.GraphKeys.REGULARIZATION_LOSSES

Input:
    name = a string prefix used for the trainable variable names
    bilm_ops = the tensorflow ops returned to compute internal
        representations from a biLM.  This is the return value
        from BidirectionalLanguageModel(...)(ids_placeholder)
    l2_coef: the l2 regularization coefficient $\lambda$.
        Pass None or 0.0 for no regularization.
    use_top_only: if True, then only use the top layer.
    do_layer_norm: if True, then apply layer normalization to each biLM
        layer before normalizing
    reuse: reuse an aggregation variable scope.

Output:
    {
        'weighted_op': op to compute weighted average for output,
        'regularization_op': op to compute regularization term
    }


if not use_character_inputs:<EOL><INDENT>if embedding_weight_file is None:<EOL><INDENT>raise ValueError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>)<EOL><DEDENT><DEDENT>self._options = options<EOL>self._weight_file = weight_file<EOL>self._embedding_weight_file = embedding_weight_file<EOL>self._use_character_inputs = use_character_inputs<EOL>self._max_batch_size = max_batch_size<EOL>self._ops = {}<EOL>self._graphs = {}<EOL>

Creates the language model computational graph and loads weights

Two options for input type:
    (1) To use character inputs (paired with Batcher)
        pass use_character_inputs=True, and ids_placeholder
        of shape (None, None, max_characters_per_token)
        to __call__
    (2) To use token ids as input (paired with TokenBatcher),
        pass use_character_inputs=False and ids_placeholder
        of shape (None, None) to __call__.
        In this case, embedding_weight_file is also required input

options_file: location of the json formatted file with
              LM hyperparameters
weight_file: location of the hdf5 file with LM weights
use_character_inputs: if True, then use character ids as input,
    otherwise use token ids
max_batch_size: the maximum allowable batch size


if ids_placeholder in self._ops:<EOL><INDENT>ret = self._ops[ids_placeholder]<EOL><DEDENT>else:<EOL><INDENT>if len(self._ops) == <NUM_LIT:0>:<EOL><INDENT>lm_graph = BidirectionalLanguageModelGraph(<EOL>self._options,<EOL>self._weight_file,<EOL>ids_placeholder,<EOL>embedding_weight_file=self._embedding_weight_file,<EOL>use_character_inputs=self._use_character_inputs,<EOL>max_batch_size=self._max_batch_size)<EOL><DEDENT>else:<EOL><INDENT>with tf.variable_scope('<STR_LIT>', reuse=True):<EOL><INDENT>lm_graph = BidirectionalLanguageModelGraph(<EOL>self._options,<EOL>self._weight_file,<EOL>ids_placeholder,<EOL>embedding_weight_file=self._embedding_weight_file,<EOL>use_character_inputs=self._use_character_inputs,<EOL>max_batch_size=self._max_batch_size)<EOL><DEDENT><DEDENT>ops = self._build_ops(lm_graph)<EOL>self._ops[ids_placeholder] = ops<EOL>self._graphs[ids_placeholder] = lm_graph<EOL>ret = ops<EOL><DEDENT>return ret<EOL>

Given the input character ids (or token ids), returns a dictionary
    with tensorflow ops:

    {'lm_embeddings': embedding_op,
     'lengths': sequence_lengths_op,
     'mask': op to compute mask}

embedding_op computes the LM embeddings and is shape
    (None, 3, None, 1024)
lengths_op computes the sequence lengths and is shape (None, )
mask computes the sequence mask and is shape (None, None)

ids_placeholder: a tf.placeholder of type int32.
    If use_character_inputs=True, it is shape
        (None, None, max_characters_per_token) and holds the input
        character ids for a batch
    If use_character_input=False, it is shape (None, None) and
        holds the input token ids for a batch


projection_dim = self.options['<STR_LIT>']['<STR_LIT>']<EOL>cnn_options = self.options['<STR_LIT>']<EOL>filters = cnn_options['<STR_LIT>']<EOL>n_filters = sum(f[<NUM_LIT:1>] for f in filters)<EOL>max_chars = cnn_options['<STR_LIT>']<EOL>char_embed_dim = cnn_options['<STR_LIT>']['<STR_LIT>']<EOL>n_chars = cnn_options['<STR_LIT>']<EOL>if n_chars != <NUM_LIT>:<EOL><INDENT>raise Exception("<STR_LIT>")<EOL><DEDENT>if cnn_options['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>activation = tf.nn.tanh<EOL><DEDENT>elif cnn_options['<STR_LIT>'] == '<STR_LIT:relu>':<EOL><INDENT>activation = tf.nn.relu<EOL><DEDENT>with tf.device("<STR_LIT>"):<EOL><INDENT>self.embedding_weights = tf.get_variable("<STR_LIT>", [n_chars, char_embed_dim],<EOL>dtype=DTYPE,<EOL>initializer=tf.random_uniform_initializer(-<NUM_LIT:1.0>, <NUM_LIT:1.0>))<EOL>self.char_embedding = tf.nn.embedding_lookup(self.embedding_weights,<EOL>self.ids_placeholder)<EOL><DEDENT>def make_convolutions(inp):<EOL><INDENT>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>convolutions = []<EOL>for i, (width, num) in enumerate(filters):<EOL><INDENT>if cnn_options['<STR_LIT>'] == '<STR_LIT:relu>':<EOL><INDENT>w_init = tf.random_uniform_initializer(<EOL>minval=-<NUM_LIT>, maxval=<NUM_LIT>)<EOL><DEDENT>elif cnn_options['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>w_init = tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>,<EOL>stddev=np.sqrt(<NUM_LIT:1.0> / (width * char_embed_dim))<EOL>)<EOL><DEDENT>w = tf.get_variable(<EOL>"<STR_LIT>" % i,<EOL>[<NUM_LIT:1>, width, char_embed_dim, num],<EOL>initializer=w_init,<EOL>dtype=DTYPE)<EOL>b = tf.get_variable(<EOL>"<STR_LIT>" % i, [num], dtype=DTYPE,<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>))<EOL>conv = tf.nn.conv2d(inp, w,<EOL>strides=[<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>],<EOL>padding="<STR_LIT>") + b<EOL>conv = tf.nn.max_pool(conv, [<NUM_LIT:1>, <NUM_LIT:1>, max_chars - width + <NUM_LIT:1>, <NUM_LIT:1>],<EOL>[<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>], '<STR_LIT>')<EOL>conv = activation(conv)<EOL>conv = tf.squeeze(conv, squeeze_dims=[<NUM_LIT:2>])<EOL>convolutions.append(conv)<EOL><DEDENT><DEDENT>return tf.concat(convolutions, <NUM_LIT:2>)<EOL><DEDENT>embedding = make_convolutions(self.char_embedding)<EOL>n_highway = cnn_options.get('<STR_LIT>')<EOL>use_highway = n_highway is not None and n_highway > <NUM_LIT:0><EOL>use_proj = n_filters != projection_dim<EOL>if use_highway or use_proj:<EOL><INDENT>batch_size_n_tokens = tf.shape(embedding)[<NUM_LIT:0>:<NUM_LIT:2>]<EOL>embedding = tf.reshape(embedding, [-<NUM_LIT:1>, n_filters])<EOL><DEDENT>if use_proj:<EOL><INDENT>assert n_filters > projection_dim<EOL>with tf.variable_scope('<STR_LIT>'):<EOL><INDENT>W_proj_cnn = tf.get_variable(<EOL>"<STR_LIT>", [n_filters, projection_dim],<EOL>initializer=tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>, stddev=np.sqrt(<NUM_LIT:1.0> / n_filters)),<EOL>dtype=DTYPE)<EOL>b_proj_cnn = tf.get_variable(<EOL>"<STR_LIT>", [projection_dim],<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>),<EOL>dtype=DTYPE)<EOL><DEDENT><DEDENT>def high(x, ww_carry, bb_carry, ww_tr, bb_tr):<EOL><INDENT>carry_gate = tf.nn.sigmoid(tf.matmul(x, ww_carry) + bb_carry)<EOL>transform_gate = tf.nn.relu(tf.matmul(x, ww_tr) + bb_tr)<EOL>return carry_gate * transform_gate + (<NUM_LIT:1.0> - carry_gate) * x<EOL><DEDENT>if use_highway:<EOL><INDENT>highway_dim = n_filters<EOL>for i in range(n_highway):<EOL><INDENT>with tf.variable_scope('<STR_LIT>' % i):<EOL><INDENT>W_carry = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim, highway_dim],<EOL>initializer=tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>, stddev=np.sqrt(<NUM_LIT:1.0> / highway_dim)),<EOL>dtype=DTYPE)<EOL>b_carry = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim],<EOL>initializer=tf.constant_initializer(-<NUM_LIT>),<EOL>dtype=DTYPE)<EOL>W_transform = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim, highway_dim],<EOL>initializer=tf.random_normal_initializer(<EOL>mean=<NUM_LIT:0.0>, stddev=np.sqrt(<NUM_LIT:1.0> / highway_dim)),<EOL>dtype=DTYPE)<EOL>b_transform = tf.get_variable(<EOL>'<STR_LIT>', [highway_dim],<EOL>initializer=tf.constant_initializer(<NUM_LIT:0.0>),<EOL>dtype=DTYPE)<EOL><DEDENT>embedding = high(embedding, W_carry, b_carry,<EOL>W_transform, b_transform)<EOL><DEDENT><DEDENT>if use_proj:<EOL><INDENT>embedding = tf.matmul(embedding, W_proj_cnn) + b_proj_cnn<EOL><DEDENT>if use_highway or use_proj:<EOL><INDENT>shp = tf.concat([batch_size_n_tokens, [projection_dim]], axis=<NUM_LIT:0>)<EOL>embedding = tf.reshape(embedding, shp)<EOL><DEDENT>self.embedding = embedding<EOL>

options contains key 'char_cnn': {

'n_characters': 262,

# includes the start / end characters
'max_characters_per_token': 50,

'filters': [
    [1, 32],
    [2, 32],
    [3, 64],
    [4, 128],
    [5, 256],
    [6, 512],
    [7, 512]
],
'activation': 'tanh',

# for the character embedding
'embedding': {'dim': 16}

# for highway layers
# if omitted, then no highway layers
'n_highway': 2,
}


def hash_(token: str, hash_size: int) -> int:

return murmurhash3_32(token, positive=True) % hash_size<EOL>

Convert a token to a hash of given size.
    Args:
        token: a word
        hash_size: hash size

    Returns:
        int, hashed token


def __call__(self, questions: List[str]) -> Sparse:

sp_tfidfs = []<EOL>for question in questions:<EOL><INDENT>ngrams = list(self.tokenizer([question]))<EOL>hashes = [hash_(ngram, self.hash_size) for ngram in ngrams[<NUM_LIT:0>]]<EOL>hashes_unique, q_hashes = np.unique(hashes, return_counts=True)<EOL>tfs = np.log1p(q_hashes)<EOL>if len(q_hashes) == <NUM_LIT:0>:<EOL><INDENT>sp_tfidfs.append(Sparse((<NUM_LIT:1>, self.hash_size)))<EOL>continue<EOL><DEDENT>size = len(self.doc_index)<EOL>Ns = self.term_freqs[hashes_unique]<EOL>idfs = np.log((size - Ns + <NUM_LIT:0.5>) / (Ns + <NUM_LIT:0.5>))<EOL>idfs[idfs < <NUM_LIT:0>] = <NUM_LIT:0><EOL>tfidf = np.multiply(tfs, idfs)<EOL>indptr = np.array([<NUM_LIT:0>, len(hashes_unique)])<EOL>sp_tfidf = Sparse((tfidf, hashes_unique, indptr), shape=(<NUM_LIT:1>, self.hash_size)<EOL>)<EOL>sp_tfidfs.append(sp_tfidf)<EOL><DEDENT>transformed = sp.sparse.vstack(sp_tfidfs)<EOL>return transformed<EOL>

Transform input list of documents to tfidf vectors.

        Args:
            questions: a list of input strings

        Returns:
            transformed documents as a csr_matrix with shape [n_documents X :attr:`hash_size`]


def get_index2doc(self) -> Dict[Any, int]:

return dict(zip(self.doc_index.values(), self.doc_index.keys()))<EOL>

Invert doc_index.

        Returns:
            inverted doc_index dict


def get_counts(self, docs: List[str], doc_ids: List[Any])-> Generator[Tuple[KeysView, ValuesView, List[int]], Any, None]:

logger.info("<STR_LIT>")<EOL>batch_ngrams = list(self.tokenizer(docs))<EOL>logger.info("<STR_LIT>")<EOL>doc_id = iter(doc_ids)<EOL>for ngrams in batch_ngrams:<EOL><INDENT>counts = Counter([hash_(gram, self.hash_size) for gram in ngrams])<EOL>hashes = counts.keys()<EOL>values = counts.values()<EOL>_id = self.doc_index[next(doc_id)]<EOL>if values:<EOL><INDENT>col_id = [_id] * len(values)<EOL><DEDENT>else:<EOL><INDENT>col_id = []<EOL><DEDENT>yield hashes, values, col_id<EOL><DEDENT>

Get term counts for a list of documents.

        Args:
            docs: a list of input documents
            doc_ids: a list of document ids corresponding to input documents

        Yields:
            a tuple of term hashes, count values and column ids

        Returns:
            None


def get_count_matrix(self, row: List[int], col: List[int], data: List[int], size: int)-> Sparse:

count_matrix = Sparse((data, (row, col)), shape=(self.hash_size, size))<EOL>count_matrix.sum_duplicates()<EOL>return count_matrix<EOL>

Get count matrix.

        Args:
            row: tfidf matrix rows corresponding to terms
            col:  tfidf matrix cols corresponding to docs
            data: tfidf matrix data corresponding to tfidf values
            size: :attr:`doc_index` size

        Returns:
            a count csr_matrix


@staticmethod<EOL><INDENT>def get_tfidf_matrix(count_matrix: Sparse) -> Tuple[Sparse, np.array]:<DEDENT>

binary = (count_matrix > <NUM_LIT:0>).astype(int)<EOL>term_freqs = np.array(binary.sum(<NUM_LIT:1>)).squeeze()<EOL>idfs = np.log((count_matrix.shape[<NUM_LIT:1>] - term_freqs + <NUM_LIT:0.5>) / (term_freqs + <NUM_LIT:0.5>))<EOL>idfs[idfs < <NUM_LIT:0>] = <NUM_LIT:0><EOL>idfs = sp.sparse.diags(idfs, <NUM_LIT:0>)<EOL>tfs = count_matrix.log1p()<EOL>tfidfs = idfs.dot(tfs)<EOL>return tfidfs, term_freqs<EOL>

Convert a count matrix into a tfidf matrix.

        Args:
            count_matrix: a count matrix

        Returns:
            a tuple of tfidf matrix and term frequences


logger.info("<STR_LIT>".format(self.save_path))<EOL>count_matrix = self.get_count_matrix(self.rows, self.cols, self.data,<EOL>size=len(self.doc_index))<EOL>tfidf_matrix, term_freqs = self.get_tfidf_matrix(count_matrix)<EOL>self.term_freqs = term_freqs<EOL>opts = {'<STR_LIT>': self.hash_size,<EOL>'<STR_LIT>': self.tokenizer.ngram_range,<EOL>'<STR_LIT>': self.doc_index,<EOL>'<STR_LIT>': self.term_freqs}<EOL>data = {<EOL>'<STR_LIT:data>': tfidf_matrix.data,<EOL>'<STR_LIT>': tfidf_matrix.indices,<EOL>'<STR_LIT>': tfidf_matrix.indptr,<EOL>'<STR_LIT>': tfidf_matrix.shape,<EOL>'<STR_LIT>': opts<EOL>}<EOL>np.savez(self.save_path, **data)<EOL>self.reset()<EOL>

Save tfidf matrix into **.npz** format.

        Returns:
            None


self.rows.clear()<EOL>self.cols.clear()<EOL>self.data.clear()<EOL>

Clear :attr:`rows`, :attr:`cols` and :attr:`data`

        Returns:
            None


if not self.load_path.exists():<EOL><INDENT>raise FileNotFoundError("<STR_LIT>")<EOL><DEDENT>logger.info("<STR_LIT>".format(self.load_path))<EOL>loader = np.load(self.load_path)<EOL>matrix = Sparse((loader['<STR_LIT:data>'], loader['<STR_LIT>'],<EOL>loader['<STR_LIT>']), shape=loader['<STR_LIT>'])<EOL>return matrix, loader['<STR_LIT>'].item(<NUM_LIT:0>)<EOL>

Load a tfidf matrix as csr_matrix.

        Returns:
            a tuple of tfidf matrix and csr data.

        Raises:
            FileNotFoundError if :attr:`load_path` doesn't exist.

        Todo:
            * implement loading from URL


def partial_fit(self, docs: List[str], doc_ids: List[Any], doc_nums: List[int]) -> None:

for doc_id, i in zip(doc_ids, doc_nums):<EOL><INDENT>self.doc_index[doc_id] = i<EOL><DEDENT>for batch_rows, batch_data, batch_cols in self.get_counts(docs, doc_ids):<EOL><INDENT>self.rows.extend(batch_rows)<EOL>self.cols.extend(batch_cols)<EOL>self.data.extend(batch_data)<EOL><DEDENT>

Partially fit on one batch.

        Args:
            docs: a list of input documents
            doc_ids: a list of document ids corresponding to input documents
            doc_nums: a list of document integer ids as they appear in a database

        Returns:
            None


def fit(self, docs: List[str], doc_ids: List[Any], doc_nums: List[int]) -> None:

self.doc_index = {}<EOL>self.rows = []<EOL>self.cols = []<EOL>self.data = []<EOL>return self.partial_fit(docs, doc_ids, doc_nums)<EOL>

Fit the vectorizer.

        Args:
            docs: a list of input documents
            doc_ids: a list of document ids corresponding to input documents
            doc_nums: a list of document integer ids as they appear in a database

        Returns:
            None


def _get_word_indexes(self, word: AnyStr) -> List:

raise NotImplementedError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL>

Transforms a word to corresponding vector of indexes


def __call__(self, data: List) -> np.ndarray:

<EOL>max_length = max(len(x) for x in data)<EOL>answer = np.zeros(shape=(len(data), max_length, self.dim), dtype=int)<EOL>for i, sent in enumerate(data):<EOL><INDENT>for j, word in enumerate(sent):<EOL><INDENT>answer[i, j][self._get_word_indexes(word)] = <NUM_LIT:1><EOL><DEDENT><DEDENT>return answer<EOL>

Transforms words to one-hot encoding according to the dictionary.

Args:
    data: the batch of words

Returns:
    a 3D array. answer[i][j][k] = 1 iff data[i][j] is the k-th word in the dictionary.


with self.save_path.open("<STR_LIT:w>", encoding="<STR_LIT:utf8>") as fout:<EOL><INDENT>for word, curr_labels in sorted(self.word_tag_mapping.items()):<EOL><INDENT>curr_labels = [self._i2t[index] for index in curr_labels]<EOL>curr_labels = [x for x in curr_labels if x != self.unk_token]<EOL>fout.write("<STR_LIT>".format(word, "<STR_LIT:U+0020>".join(curr_labels)))<EOL><DEDENT><DEDENT>


if not isinstance(self.load_path, list):<EOL><INDENT>self.load_path = [self.load_path]<EOL><DEDENT>for i, path in enumerate(self.load_path):<EOL><INDENT>if isinstance(path, str):<EOL><INDENT>self.load_path[i] = pathlib.Path(path)<EOL><DEDENT><DEDENT>labels_by_words = defaultdict(set)<EOL>for infile in self.load_path:<EOL><INDENT>with infile.open("<STR_LIT:r>", encoding="<STR_LIT:utf8>") as fin:<EOL><INDENT>for line in fin:<EOL><INDENT>line = line.strip()<EOL>if line.count("<STR_LIT:\t>") != <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>word, labels = line.split("<STR_LIT:\t>")<EOL>labels_by_words[word].update(labels.split())<EOL><DEDENT><DEDENT><DEDENT>self._initialize(labels_by_words)<EOL>

Loads the dictionary from self.load_path


with self.save_path.open("<STR_LIT:w>", encoding="<STR_LIT:utf8>") as fout:<EOL><INDENT>fout.write("<STR_LIT:\n>".join(self._i2t))<EOL><DEDENT>


self._i2t = []<EOL>with self.load_path.open("<STR_LIT:r>", encoding="<STR_LIT:utf8>") as fin:<EOL><INDENT>for line in fin:<EOL><INDENT>line = line.strip()<EOL>if line == "<STR_LIT>":<EOL><INDENT>continue<EOL><DEDENT>self._i2t.append(line)<EOL><DEDENT><DEDENT>self._t2i = {tag: i for i, tag in enumerate(self._i2t)}<EOL>self._make_tag_trie()<EOL>


def find_compatible(self, tag: str) -> List[int]:

if "<STR_LIT:U+0020>" in tag and "<STR_LIT:_>" not in tag:<EOL><INDENT>pos, tag = tag.split("<STR_LIT:U+0020>", maxsplit=<NUM_LIT:1>)<EOL>tag = sorted([tuple(elem.split("<STR_LIT:=>")) for elem in tag.split("<STR_LIT:|>")])<EOL><DEDENT>else:<EOL><INDENT>pos, tag = tag.split()[<NUM_LIT:0>], []<EOL><DEDENT>if pos not in self._start_nodes_for_pos:<EOL><INDENT>return []<EOL><DEDENT>tag = [(key, self.VALUE_MAP.get(value, value)) for key, value in tag<EOL>if key not in self.USELESS_KEYS]<EOL>if len(tag) > <NUM_LIT:0>:<EOL><INDENT>curr_nodes = [(<NUM_LIT:0>, self._start_nodes_for_pos[pos])]<EOL>final_nodes = []<EOL><DEDENT>else:<EOL><INDENT>final_nodes = [self._start_nodes_for_pos[pos]]<EOL>curr_nodes = []<EOL><DEDENT>while len(curr_nodes) > <NUM_LIT:0>:<EOL><INDENT>i, node_index = curr_nodes.pop()<EOL>node = self._nodes[node_index]<EOL>if len(node) == <NUM_LIT:0>:<EOL><INDENT>final_nodes.append(node_index)<EOL><DEDENT>for curr_key, curr_values_dict in node.items():<EOL><INDENT>curr_i, curr_node_index = i, node_index<EOL>while curr_i < len(tag) and tag[curr_i][<NUM_LIT:0>] < curr_key:<EOL><INDENT>curr_i += <NUM_LIT:1><EOL><DEDENT>if curr_i == len(tag):<EOL><INDENT>final_nodes.extend(curr_values_dict.values())<EOL>continue<EOL><DEDENT>key, value = tag[curr_i]<EOL>if curr_key < key:<EOL><INDENT>for child in curr_values_dict.values():<EOL><INDENT>curr_nodes.append((curr_i, child))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>child = curr_values_dict.get(value)<EOL>if child is not None:<EOL><INDENT>if curr_i < len(tag) - <NUM_LIT:1>:<EOL><INDENT>curr_nodes.append((curr_i + <NUM_LIT:1>, child))<EOL><DEDENT>else:<EOL><INDENT>final_nodes.append(child)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>answer = []<EOL>while len(final_nodes) > <NUM_LIT:0>:<EOL><INDENT>index = final_nodes.pop()<EOL>if self._data[index] is not None:<EOL><INDENT>answer.append(self._data[index])<EOL><DEDENT>for elem in self._nodes[index].values():<EOL><INDENT>final_nodes.extend(elem.values())<EOL><DEDENT><DEDENT>return answer<EOL>

Transforms a Pymorphy tag to a list of indexes of compatible UD tags.

Args:
    tag: input Pymorphy tag

Returns:
    indexes of compatible UD tags


m, n = len(needle), len(haystack)<EOL>if m == <NUM_LIT:1>:<EOL><INDENT>return needle not in haystack<EOL><DEDENT>if not n:<EOL><INDENT>return m<EOL><DEDENT>row1 = [<NUM_LIT:0>] * (n + <NUM_LIT:1>)<EOL>for j in range(<NUM_LIT:0>, n + <NUM_LIT:1>):<EOL><INDENT>if j == <NUM_LIT:0> or not haystack[j - <NUM_LIT:1>].isalnum():<EOL><INDENT>row1[j] = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>row1[j] = row1[j - <NUM_LIT:1>] + <NUM_LIT:1><EOL><DEDENT><DEDENT>for i in range(<NUM_LIT:0>, m):<EOL><INDENT>row2 = [i + <NUM_LIT:1>]<EOL>for j in range(<NUM_LIT:0>, n):<EOL><INDENT>cost = (needle[i] != haystack[j])<EOL>row2.append(min(row1[j + <NUM_LIT:1>] + <NUM_LIT:1>, row2[j] + <NUM_LIT:1>, row1[j] + cost))<EOL><DEDENT>row1 = row2<EOL><DEDENT>d = n + m<EOL>j_min = <NUM_LIT:0><EOL>for j in range(<NUM_LIT:0>, n + <NUM_LIT:1>):<EOL><INDENT>if j == <NUM_LIT:0> or j == n or not haystack[j].isalnum():<EOL><INDENT>if d > row1[j]:<EOL><INDENT>d = row1[j]<EOL>j_min = j<EOL><DEDENT><DEDENT><DEDENT>return d, j_min<EOL>

Calculates the fuzzy match of needle in haystack,
        using a modified version of the Levenshtein distance
        algorithm.
        The function is modified from the Levenshtein function
        in the bktree module by Adam Hupp
        :type needle: string
        :type haystack: string


def __call__(self, questions: List[str]) -> Tuple[List[Any], List[float]]:

batch_doc_ids, batch_docs_scores = [], []<EOL>q_tfidfs = self.vectorizer(questions)<EOL>for q_tfidf in q_tfidfs:<EOL><INDENT>scores = q_tfidf * self.vectorizer.tfidf_matrix<EOL>scores = np.squeeze(<EOL>scores.toarray() + <NUM_LIT>)  <EOL>if self.active:<EOL><INDENT>thresh = self.top_n<EOL><DEDENT>else:<EOL><INDENT>thresh = len(self.vectorizer.doc_index)<EOL><DEDENT>if thresh >= len(scores):<EOL><INDENT>o = np.argpartition(-scores, len(scores) - <NUM_LIT:1>)[<NUM_LIT:0>:thresh]<EOL><DEDENT>else:<EOL><INDENT>o = np.argpartition(-scores, thresh)[<NUM_LIT:0>:thresh]<EOL><DEDENT>o_sort = o[np.argsort(-scores[o])]<EOL>doc_scores = scores[o_sort]<EOL>doc_ids = [self.vectorizer.index2doc[i] for i in o_sort]<EOL>batch_doc_ids.append(doc_ids)<EOL>batch_docs_scores.append(doc_scores)<EOL><DEDENT>return batch_doc_ids, batch_docs_scores<EOL>

Rank documents and return top n document titles with scores.

        Args:
            questions: list of queries used in ranking

        Returns:
            a tuple of selected doc ids and their scores


def __call__(self, contexts_batch: List[List[str]], questions_batch: List[List[str]]) ->Tuple[List[str], List[float]]:

<EOL>warnings.warn(f'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>', FutureWarning)<EOL>batch_best_answers = []<EOL>batch_best_answers_scores = []<EOL>for contexts, questions in zip(contexts_batch, questions_batch):<EOL><INDENT>results = []<EOL>for i in range(<NUM_LIT:0>, len(contexts), self.batch_size):<EOL><INDENT>c_batch = contexts[i: i + self.batch_size]<EOL>q_batch = questions[i: i + self.batch_size]<EOL>batch_predict = zip(*self.squad_model(c_batch, q_batch))<EOL>results += batch_predict<EOL><DEDENT>if self.sort_noans:<EOL><INDENT>results = sorted(results, key=lambda x: (x[<NUM_LIT:0>] != '<STR_LIT>', x[<NUM_LIT:2>]), reverse=True)<EOL><DEDENT>else:<EOL><INDENT>results = sorted(results, key=itemgetter(<NUM_LIT:2>), reverse=True)<EOL><DEDENT>batch_best_answers.append(results[<NUM_LIT:0>][<NUM_LIT:0>])<EOL>batch_best_answers_scores.append(results[<NUM_LIT:0>][<NUM_LIT:2>])<EOL><DEDENT>return batch_best_answers, batch_best_answers_scores<EOL>

Sort obtained results from squad reader by logits and get the answer with a maximum logit.

Args:
    contexts_batch: a batch of contexts which should be treated as a single batch in the outer JSON config
    questions_batch: a batch of questions which should be treated as a single batch in the outer JSON config

Returns:
    a batch of best answers and their scores


def __call__(self, input_doc_ids: List[List[Any]], input_doc_scores: List[List[float]]) ->Tuple[List[List], List[List]]:

batch_ids = []<EOL>batch_scores = []<EOL>for instance_ids, instance_scores in zip(input_doc_ids, input_doc_scores):<EOL><INDENT>instance_probas = []<EOL>for idx, score in zip(instance_ids, instance_scores):<EOL><INDENT>pop = self.pop_dict.get(idx, self.mean_pop)<EOL>features = [score, pop, score * pop]<EOL>prob = self.clf.predict_proba([features])<EOL>instance_probas.append(prob[<NUM_LIT:0>][<NUM_LIT:1>])<EOL><DEDENT>sort = sorted(enumerate(instance_probas), key=itemgetter(<NUM_LIT:1>), reverse=True)<EOL>sorted_probas = [item[<NUM_LIT:1>] for item in sort]<EOL>sorted_ids = [instance_ids[item[<NUM_LIT:0>]] for item in sort]<EOL>if self.active:<EOL><INDENT>sorted_ids = sorted_ids[:self.top_n]<EOL>sorted_probas = sorted_probas[:self.top_n]<EOL><DEDENT>batch_ids.append(sorted_ids)<EOL>batch_scores.append(sorted_probas)<EOL><DEDENT>return batch_ids, batch_scores<EOL>

Get tfidf scores and tfidf ids, re-rank them by applying logistic regression classifier,
        output pop ranker ids and pop ranker scores.

         Args:
            input_doc_ids: top input doc ids of tfidf ranker
            input_doc_scores: top input doc scores of tfidf ranker corresponding to doc ids

        Returns:
            top doc ids of pop ranker and their corresponding scores


return hash(self.text)<EOL>

Override the default hash behavior (that returns the id)


return hash(self.default + '<STR_LIT:\t>' + self.dontcare)<EOL>


if isinstance(key, str):<EOL><INDENT>return key in self.act2templ<EOL><DEDENT>elif isinstance(key, Template):<EOL><INDENT>return key in self.templ2act<EOL><DEDENT>

If key is an str, returns whether the key is in the actions.
        If key is a Template, returns if the key is templates.


if isinstance(key, str):<EOL><INDENT>return self.act2templ[key]<EOL><DEDENT>elif isinstance(key, Template):<EOL><INDENT>return self.templ2act[key]<EOL><DEDENT>

If key is an str, returns corresponding template.
        If key is a Template, return corresponding action.
        If does not exist, return None.


key = str(key)<EOL>if key not in self.act2templ:<EOL><INDENT>self.act2templ[key] = value<EOL>self.templ2act[value] = key<EOL>self._actions = []<EOL>self._templates = []<EOL><DEDENT>

If the key is not in  the dictionary, add it.


@abstractmethod<EOL><INDENT>def reset_state(self) -> None:<DEDENT>

pass<EOL>


@abstractmethod<EOL><INDENT>def update_state(self,<EOL>slots: Union[List[Tuple[str, Any]], Dict[str, Any]]) -> '<STR_LIT>':<DEDENT>

pass<EOL>

Updates dialogue state with new ``slots``, calculates features.

Returns:
    Tracker: .


@abstractmethod<EOL><INDENT>def get_state(self) -> Dict[str, Any]:<DEDENT>

pass<EOL>

Returns:
    Dict[str, Any]: dictionary with current slots and their values.


@abstractmethod<EOL><INDENT>def get_features(self) -> np.ndarray:<DEDENT>

pass<EOL>

Returns:
    np.ndarray[float]: numpy array with calculates state features.


template = self.templates.templates[int(action_id)]<EOL>slots = self.tracker.get_state()<EOL>if self.db_result is not None:<EOL><INDENT>for k, v in self.db_result.items():<EOL><INDENT>slots[k] = str(v)<EOL><DEDENT><DEDENT>resp = template.generate_text(slots)<EOL>if (self.templates.ttype is templ.DualTemplate) and(action_id == self.api_call_id):<EOL><INDENT>resp = re.sub("<STR_LIT>", "<STR_LIT>", resp).lower()<EOL><DEDENT>if self.debug:<EOL><INDENT>log.debug("<STR_LIT>".format(resp))<EOL><DEDENT>return resp<EOL>

Convert action template id and entities from tracker
to final response.


def __init__(self, embedding_size: int, n_classes: int,<EOL>model_name: str, optimizer: str = "<STR_LIT>", loss: str = "<STR_LIT>",<EOL>learning_rate: Union[None, float, List[float]] = None,<EOL>learning_rate_decay: Optional[Union[float, str]] = <NUM_LIT:0.>,<EOL>last_layer_activation: str = "<STR_LIT>",<EOL>restore_lr: bool = False,<EOL>classes: Optional[Union[list, Generator]] = None,<EOL>text_size: Optional[int] = None,<EOL>padding: Optional[str] = "<STR_LIT>",<EOL>**kwargs):

if learning_rate is None and isinstance(learning_rate_decay, float):<EOL><INDENT>learning_rate = <NUM_LIT><EOL><DEDENT>elif learning_rate is None and learning_rate_decay is None:<EOL><INDENT>learning_rate = <NUM_LIT><EOL>learning_rate_decay = <NUM_LIT:0.><EOL><DEDENT>elif isinstance(learning_rate, float) and "<STR_LIT>" in kwargs:<EOL><INDENT>learning_rate_decay = "<STR_LIT>"<EOL><DEDENT>if classes is not None:<EOL><INDENT>classes = list(classes)<EOL><DEDENT>given_opt = {"<STR_LIT>": embedding_size,<EOL>"<STR_LIT>": n_classes,<EOL>"<STR_LIT>": model_name,<EOL>"<STR_LIT>": optimizer,<EOL>"<STR_LIT>": loss,<EOL>"<STR_LIT>": learning_rate,<EOL>"<STR_LIT>": learning_rate_decay,<EOL>"<STR_LIT>": last_layer_activation,<EOL>"<STR_LIT>": restore_lr,<EOL>"<STR_LIT>": classes,<EOL>"<STR_LIT>": text_size,<EOL>"<STR_LIT>": padding,<EOL>**kwargs}<EOL>self.opt = deepcopy(given_opt)<EOL>self.model = None<EOL>super().__init__(**given_opt)<EOL>if classes is not None:<EOL><INDENT>self.classes = self.opt.get("<STR_LIT>")<EOL><DEDENT>self.n_classes = self.opt.get('<STR_LIT>')<EOL>if self.n_classes == <NUM_LIT:0>:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>self.load()<EOL>summary = ['<STR_LIT>', '<STR_LIT>']<EOL>self.model.summary(print_fn=summary.append)<EOL>log.info('<STR_LIT:\n>'.join(summary))<EOL>

Initialize model using parameters
from opt dictionary (from config), if model is being initialized from saved.


def pad_texts(self, sentences: List[List[np.ndarray]]) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:

pad = np.zeros(self.opt['<STR_LIT>'])<EOL>cutted_batch = [sen[:self.opt['<STR_LIT>']] for sen in sentences]<EOL>if self.opt["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>cutted_batch = [[pad] * (self.opt['<STR_LIT>'] - len(tokens)) + list(tokens) for tokens in cutted_batch]<EOL><DEDENT>elif self.opt["<STR_LIT>"] == "<STR_LIT>":<EOL><INDENT>cutted_batch = [list(tokens) + [pad] * (self.opt['<STR_LIT>'] - len(tokens)) for tokens in cutted_batch]<EOL><DEDENT>else:<EOL><INDENT>raise ConfigError("<STR_LIT>".format(self.opt['<STR_LIT>']))<EOL><DEDENT>return np.asarray(cutted_batch)<EOL>

Cut and pad tokenized texts to self.opt["text_size"] tokens

Args:
    sentences: list of lists of tokens

Returns:
    array of embedded texts


def check_input(self, texts: List[List[np.ndarray]]) -> np.ndarray:

if self.opt["<STR_LIT>"] is not None:<EOL><INDENT>features = self.pad_texts(texts)<EOL><DEDENT>else:<EOL><INDENT>if len(texts[<NUM_LIT:0>]):<EOL><INDENT>features = np.array(texts)<EOL><DEDENT>else:<EOL><INDENT>features = np.zeros((<NUM_LIT:1>, <NUM_LIT:1>, self.opt["<STR_LIT>"]))<EOL><DEDENT><DEDENT>return features<EOL>

Check and convert input to array of tokenized embedded samples

Args:
    texts: list of tokenized embedded text samples

Returns:
    array of tokenized embedded texts samples that are cut and padded


def train_on_batch(self, texts: List[List[np.ndarray]], labels: list) -> Union[float, List[float]]:

features = self.check_input(texts)<EOL>metrics_values = self.model.train_on_batch(features, np.squeeze(np.array(labels)))<EOL>return metrics_values<EOL>

Train the model on the given batch

Args:
    texts: list of tokenized embedded text samples
    labels: list of labels

Returns:
    metrics values on the given batch


def infer_on_batch(self, texts: List[List[np.ndarray]], labels: list = None) ->Union[float, List[float], np.ndarray]:

features = self.check_input(texts)<EOL>if labels:<EOL><INDENT>metrics_values = self.model.test_on_batch(features, np.squeeze(np.array(labels)))<EOL>return metrics_values<EOL><DEDENT>else:<EOL><INDENT>predictions = self.model.predict(features)<EOL>return predictions<EOL><DEDENT>

Infer the model on the given batch

Args:
    texts: list of tokenized embedded text samples
    labels: list of labels

Returns:
    metrics values on the given batch, if labels are given
    predictions, otherwise


def __call__(self, data: List[List[np.ndarray]], *args) -> List[List[float]]:

preds = np.array(self.infer_on_batch(data), dtype="<STR_LIT>").tolist()<EOL>return preds<EOL>

Infer on the given data

Args:
    data: list of tokenized text samples
    *args: additional arguments

Returns:
    for each sentence:
        vector of probabilities to belong with each class
        or list of labels sentence belongs with


def init_model_from_scratch(self, model_name: str) -> Model:

log.info(f'<STR_LIT>')<EOL>model_func = getattr(self, model_name, None)<EOL>if callable(model_func):<EOL><INDENT>model = model_func(**self.opt)<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError("<STR_LIT>".format(model_name))<EOL><DEDENT>return model<EOL>

Initialize uncompiled model from scratch with given params

Args:
    model_name: name of model function described as a method of this class

Returns:
    compiled model with given network and learning parameters


def _load(self, model_name: str) -> None:

if self.load_path:<EOL><INDENT>if isinstance(self.load_path, Path) and not self.load_path.parent.is_dir():<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>opt_path = Path("<STR_LIT>".format(str(self.load_path.resolve())))<EOL>weights_path = Path("<STR_LIT>".format(str(self.load_path.resolve())))<EOL>if opt_path.exists() and weights_path.exists():<EOL><INDENT>log.info("<STR_LIT>".format(self.__class__.__name__))<EOL>self.opt["<STR_LIT>"] = read_json(opt_path).get("<STR_LIT>")<EOL>model_func = getattr(self, model_name, None)<EOL>if callable(model_func):<EOL><INDENT>model = model_func(**self.opt)<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError("<STR_LIT>".format(model_name))<EOL><DEDENT>log.info("<STR_LIT>".format(weights_path.name))<EOL>try:<EOL><INDENT>model.load_weights(str(weights_path))<EOL><DEDENT>except ValueError:<EOL><INDENT>ConfigError("<STR_LIT>")<EOL><DEDENT>self.model = model<EOL>return None<EOL><DEDENT>else:<EOL><INDENT>self.model = self.init_model_from_scratch(model_name)<EOL>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.warning("<STR_LIT>".format(self.__class__.__name__))<EOL>self.model = self.init_model_from_scratch(model_name)<EOL>return None<EOL><DEDENT>

Initialize uncompiled model from saved params and weights

Args:
    model_name: name of model function described as a method of this class

Returns:
    model with loaded weights and network parameters from files
    but compiled with given learning parameters


def compile(self, model: Model, optimizer_name: str, loss_name: str,<EOL>learning_rate: Optional[Union[float, List[float]]],<EOL>learning_rate_decay: Optional[Union[float, str]]) -> Model:

optimizer_func = getattr(keras.optimizers, optimizer_name, None)<EOL>if callable(optimizer_func):<EOL><INDENT>if isinstance(learning_rate, float) and isinstance(learning_rate_decay, float):<EOL><INDENT>self.optimizer = optimizer_func(lr=learning_rate, decay=learning_rate_decay)<EOL><DEDENT>else:<EOL><INDENT>self.optimizer = optimizer_func()<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise AttributeError("<STR_LIT>".format(optimizer_name))<EOL><DEDENT>loss_func = getattr(keras.losses, loss_name, None)<EOL>if callable(loss_func):<EOL><INDENT>loss = loss_func<EOL><DEDENT>else:<EOL><INDENT>raise AttributeError("<STR_LIT>".format(loss_name))<EOL><DEDENT>model.compile(optimizer=self.optimizer,<EOL>loss=loss)<EOL>return model<EOL>

Compile model with given optimizer and loss

Args:
    model: keras uncompiled model
    optimizer_name: name of optimizer from keras.optimizers
    loss_name: loss function name (from keras.losses)
    learning_rate: learning rate.
    learning_rate_decay: learning rate decay.

Returns:


@overrides<EOL><INDENT>def save(self, fname: str = None) -> None:<DEDENT>

if not fname:<EOL><INDENT>fname = self.save_path<EOL><DEDENT>else:<EOL><INDENT>fname = Path(fname).resolve()<EOL><DEDENT>if not fname.parent.is_dir():<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>else:<EOL><INDENT>opt_path = f"<STR_LIT>"<EOL>weights_path = f"<STR_LIT>"<EOL>log.info(f"<STR_LIT>")<EOL>self.model.save_weights(weights_path)<EOL><DEDENT>self.opt["<STR_LIT>"] = self.epochs_done<EOL>if isinstance(self.opt.get("<STR_LIT>", None), float):<EOL><INDENT>self.opt["<STR_LIT>"] = K.eval(self.optimizer.lr) / (<NUM_LIT:1.> +<EOL>K.eval(self.optimizer.decay) * self.batches_seen)<EOL><DEDENT>if self.opt.get("<STR_LIT>") and self.opt.get("<STR_LIT>"):<EOL><INDENT>if self.opt.get("<STR_LIT>") != self.opt.get("<STR_LIT>"):<EOL><INDENT>self.opt["<STR_LIT>"] = str(self.opt["<STR_LIT>"])<EOL><DEDENT><DEDENT>save_json(self.opt, opt_path)<EOL>

Save the model parameters into <<fname>>_opt.json (or <<ser_file>>_opt.json)
and model weights into <<fname>>.h5 (or <<ser_file>>.h5)
Args:
    fname: file_path to save model. If not explicitly given seld.opt["ser_file"] will be used

Returns:
    None


def cnn_model(self, kernel_sizes_cnn: List[int], filters_cnn: int, dense_size: int,<EOL>coef_reg_cnn: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>, dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>outputs = []<EOL>for i in range(len(kernel_sizes_cnn)):<EOL><INDENT>output_i = Conv1D(filters_cnn, kernel_size=kernel_sizes_cnn[i],<EOL>activation=None,<EOL>kernel_regularizer=l2(coef_reg_cnn),<EOL>padding='<STR_LIT>')(output)<EOL>output_i = BatchNormalization()(output_i)<EOL>output_i = Activation('<STR_LIT:relu>')(output_i)<EOL>output_i = GlobalMaxPooling1D()(output_i)<EOL>outputs.append(output_i)<EOL><DEDENT>output = concatenate(outputs, axis=<NUM_LIT:1>)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled model of shallow-and-wide CNN.

Args:
    kernel_sizes_cnn: list of kernel sizes of convolutions.
    filters_cnn: number of filters for convolutions.
    dense_size: number of units for dense layer.
    coef_reg_cnn: l2-regularization coefficient for convolutions.
    coef_reg_den: l2-regularization coefficient for dense layers.
    dropout_rate: dropout rate used after convolutions and between dense layers.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def dcnn_model(self, kernel_sizes_cnn: List[int], filters_cnn: List[int], dense_size: int,<EOL>coef_reg_cnn: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>, dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>for i in range(len(kernel_sizes_cnn)):<EOL><INDENT>output = Conv1D(filters_cnn[i], kernel_size=kernel_sizes_cnn[i],<EOL>activation=None,<EOL>kernel_regularizer=l2(coef_reg_cnn),<EOL>padding='<STR_LIT>')(output)<EOL>output = BatchNormalization()(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = MaxPooling1D()(output)<EOL><DEDENT>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled model of deep CNN.

Args:
    kernel_sizes_cnn: list of kernel sizes of convolutions.
    filters_cnn: number of filters for convolutions.
    dense_size: number of units for dense layer.
    coef_reg_cnn: l2-regularization coefficient for convolutions.
    coef_reg_den: l2-regularization coefficient for dense layers.
    dropout_rate: dropout rate used after convolutions and between dense layers.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def cnn_model_max_and_aver_pool(self, kernel_sizes_cnn: List[int], filters_cnn: int, dense_size: int,<EOL>coef_reg_cnn: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>, dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>outputs = []<EOL>for i in range(len(kernel_sizes_cnn)):<EOL><INDENT>output_i = Conv1D(filters_cnn, kernel_size=kernel_sizes_cnn[i],<EOL>activation=None,<EOL>kernel_regularizer=l2(coef_reg_cnn),<EOL>padding='<STR_LIT>')(output)<EOL>output_i = BatchNormalization()(output_i)<EOL>output_i = Activation('<STR_LIT:relu>')(output_i)<EOL>output_i_0 = GlobalMaxPooling1D()(output_i)<EOL>output_i_1 = GlobalAveragePooling1D()(output_i)<EOL>output_i = Concatenate()([output_i_0, output_i_1])<EOL>outputs.append(output_i)<EOL><DEDENT>output = concatenate(outputs, axis=<NUM_LIT:1>)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = BatchNormalization()(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled model of shallow-and-wide CNN where average pooling after convolutions is replaced with
concatenation of average and max poolings.

Args:
    kernel_sizes_cnn: list of kernel sizes of convolutions.
    filters_cnn: number of filters for convolutions.
    dense_size: number of units for dense layer.
    coef_reg_cnn: l2-regularization coefficient for convolutions. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate used after convolutions and between dense layers. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bilstm_model(self, units_lstm: int, dense_size: int,<EOL>coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>output = Bidirectional(LSTM(units_lstm, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled BiLSTM.

Args:
    units_lstm (int): number of units for LSTM.
    dense_size (int): number of units for dense layer.
    coef_reg_lstm (float): l2-regularization coefficient for LSTM. Default: ``0.0``.
    coef_reg_den (float): l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate (float): dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
    rec_dropout_rate (float): dropout rate for LSTM. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bilstm_bilstm_model(self, units_lstm_1: int, units_lstm_2: int, dense_size: int,<EOL>coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>output = Bidirectional(LSTM(units_lstm_1, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Bidirectional(LSTM(units_lstm_2, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled two-layers BiLSTM.

Args:
    units_lstm_1: number of units for the first LSTM layer.
    units_lstm_2: number of units for the second LSTM layer.
    dense_size: number of units for dense layer.
    coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bilstm_cnn_model(self, units_lstm: int, kernel_sizes_cnn: List[int], filters_cnn: int, dense_size: int,<EOL>coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_cnn: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>output = Bidirectional(LSTM(units_lstm, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = Reshape(target_shape=(self.opt['<STR_LIT>'], <NUM_LIT:2> * units_lstm))(output)<EOL>outputs = []<EOL>for i in range(len(kernel_sizes_cnn)):<EOL><INDENT>output_i = Conv1D(filters_cnn,<EOL>kernel_size=kernel_sizes_cnn[i],<EOL>activation=None,<EOL>kernel_regularizer=l2(coef_reg_cnn),<EOL>padding='<STR_LIT>')(output)<EOL>output_i = BatchNormalization()(output_i)<EOL>output_i = Activation('<STR_LIT:relu>')(output_i)<EOL>output_i = GlobalMaxPooling1D()(output_i)<EOL>outputs.append(output_i)<EOL><DEDENT>output = Concatenate(axis=<NUM_LIT:1>)(outputs)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled BiLSTM-CNN.

Args:
    units_lstm: number of units for LSTM.
    kernel_sizes_cnn: list of kernel sizes of convolutions.
    filters_cnn: number of filters for convolutions.
    dense_size: number of units for dense layer.
    coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``.
    coef_reg_cnn: l2-regularization coefficient for convolutions. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def cnn_bilstm_model(self, kernel_sizes_cnn: List[int], filters_cnn: int, units_lstm: int, dense_size: int,<EOL>coef_reg_cnn: float = <NUM_LIT:0.>, coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>outputs = []<EOL>for i in range(len(kernel_sizes_cnn)):<EOL><INDENT>output_i = Conv1D(filters_cnn, kernel_size=kernel_sizes_cnn[i],<EOL>activation=None,<EOL>kernel_regularizer=l2(coef_reg_cnn),<EOL>padding='<STR_LIT>')(output)<EOL>output_i = BatchNormalization()(output_i)<EOL>output_i = Activation('<STR_LIT:relu>')(output_i)<EOL>output_i = MaxPooling1D()(output_i)<EOL>outputs.append(output_i)<EOL><DEDENT>output = concatenate(outputs, axis=<NUM_LIT:1>)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Bidirectional(LSTM(units_lstm, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Build un-compiled BiLSTM-CNN.

Args:
    kernel_sizes_cnn: list of kernel sizes of convolutions.
    filters_cnn: number of filters for convolutions.
    units_lstm: number of units for LSTM.
    dense_size: number of units for dense layer.
    coef_reg_cnn: l2-regularization coefficient for convolutions. Default: ``0.0``.
    coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bilstm_self_add_attention_model(self, units_lstm: int, dense_size: int, self_att_hid: int, self_att_out: int,<EOL>coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>output = Bidirectional(LSTM(units_lstm, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = MaxPooling1D(pool_size=<NUM_LIT:2>, strides=<NUM_LIT:3>)(output)<EOL>output = additive_self_attention(output, n_hidden=self_att_hid,<EOL>n_output_features=self_att_out)<EOL>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Method builds uncompiled model of BiLSTM with self additive attention.

Args:
    units_lstm: number of units for LSTM.
    self_att_hid: number of hidden units in self-attention
    self_att_out: number of output units in self-attention
    dense_size: number of units for dense layer.
    coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bilstm_self_mult_attention_model(self, units_lstm: int, dense_size: int, self_att_hid: int, self_att_out: int,<EOL>coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>output = Bidirectional(LSTM(units_lstm, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = MaxPooling1D(pool_size=<NUM_LIT:2>, strides=<NUM_LIT:3>)(output)<EOL>output = multiplicative_self_attention(output, n_hidden=self_att_hid,<EOL>n_output_features=self_att_out)<EOL>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Method builds uncompiled model of BiLSTM with self multiplicative attention.

Args:
    units_lstm: number of units for LSTM.
    self_att_hid: number of hidden units in self-attention
    self_att_out: number of output units in self-attention
    dense_size: number of units for dense layer.
    coef_reg_lstm: l2-regularization coefficient for LSTM. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiLSTM and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for LSTM. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bigru_model(self, units_gru: int, dense_size: int,<EOL>coef_reg_lstm: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>input_projection_size: Optional[int] = None, **kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = inp<EOL>if input_projection_size is not None:<EOL><INDENT>output = Dense(input_projection_size, activation='<STR_LIT:relu>')(output)<EOL><DEDENT>output = Bidirectional(GRU(units_gru, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>kernel_regularizer=l2(coef_reg_lstm),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output = GlobalMaxPooling1D()(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Method builds uncompiled model BiGRU.

Args:
    units_gru: number of units for GRU.
    dense_size: number of units for dense layer.
    coef_reg_lstm: l2-regularization coefficient for GRU. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiGRU and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for GRU. Default: ``0.0``.
    input_projection_size: if not None, adds Dense layer (with ``relu`` activation)
                           right after input layer to the size ``input_projection_size``.
                           Useful for input dimentionaliry recuction. Default: ``None``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def bigru_with_max_aver_pool_model(self, units_gru: int, dense_size: int,<EOL>coef_reg_gru: float = <NUM_LIT:0.>, coef_reg_den: float = <NUM_LIT:0.>,<EOL>dropout_rate: float = <NUM_LIT:0.>, rec_dropout_rate: float = <NUM_LIT:0.>,<EOL>**kwargs) -> Model:

inp = Input(shape=(self.opt['<STR_LIT>'], self.opt['<STR_LIT>']))<EOL>output = Dropout(rate=dropout_rate)(inp)<EOL>output, state1, state2 = Bidirectional(GRU(units_gru, activation='<STR_LIT>',<EOL>return_sequences=True,<EOL>return_state=True,<EOL>kernel_regularizer=l2(coef_reg_gru),<EOL>dropout=dropout_rate,<EOL>recurrent_dropout=rec_dropout_rate))(output)<EOL>output1 = GlobalMaxPooling1D()(output)<EOL>output2 = GlobalAveragePooling1D()(output)<EOL>output = Concatenate()([output1, output2, state1, state2])<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(dense_size, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>output = Activation('<STR_LIT:relu>')(output)<EOL>output = Dropout(rate=dropout_rate)(output)<EOL>output = Dense(self.n_classes, activation=None,<EOL>kernel_regularizer=l2(coef_reg_den))(output)<EOL>act_output = Activation(self.opt.get("<STR_LIT>", "<STR_LIT>"))(output)<EOL>model = Model(inputs=inp, outputs=act_output)<EOL>return model<EOL>

Method builds uncompiled model Bidirectional GRU with concatenation of max and average pooling after BiGRU.

Args:
    units_gru: number of units for GRU.
    dense_size: number of units for dense layer.
    coef_reg_gru: l2-regularization coefficient for GRU. Default: ``0.0``.
    coef_reg_den: l2-regularization coefficient for dense layers. Default: ``0.0``.
    dropout_rate: dropout rate to be used after BiGRU and between dense layers. Default: ``0.0``.
    rec_dropout_rate: dropout rate for GRU. Default: ``0.0``.
    kwargs: other non-used parameters

Returns:
    keras.models.Model: uncompiled instance of Keras Model


def __init__(self,<EOL>max_proba: bool = None,<EOL>confident_threshold: float = None,<EOL>top_n: int = None,<EOL>**kwargs) -> None:

self.max_proba = max_proba<EOL>self.confident_threshold = confident_threshold<EOL>self.top_n = top_n<EOL>


def __call__(self, data: Union[np.ndarray, List[List[float]], List[List[int]]],<EOL>*args, **kwargs) -> Union[List[List[str]], List[str]]:

if self.confident_threshold:<EOL><INDENT>return [list(np.where(np.array(d) > self.confident_threshold)[<NUM_LIT:0>])<EOL>for d in data]<EOL><DEDENT>elif self.max_proba:<EOL><INDENT>return [[np.argmax(d)] for d in data]<EOL><DEDENT>elif self.top_n:<EOL><INDENT>return [np.argsort(d)[::-<NUM_LIT:1>][:self.top_n] for d in data]<EOL><DEDENT>else:<EOL><INDENT>raise ConfigError("<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>

Process probabilities to labels

Args:
    data: list of vectors with probability distribution
    *args:
    **kwargs:

Returns:
    list of labels (only label classification) or list of lists of labels (multi-label classification)


def labels2onehot(labels: [List[str], List[List[str]], np.ndarray], classes:  [list, np.ndarray]) -> np.ndarray:

n_classes = len(classes)<EOL>y = []<EOL>for sample in labels:<EOL><INDENT>curr = np.zeros(n_classes)<EOL>if isinstance(sample, list):<EOL><INDENT>for intent in sample:<EOL><INDENT>if intent not in classes:<EOL><INDENT>log.warning('<STR_LIT>'.format(intent))<EOL><DEDENT>else:<EOL><INDENT>curr[np.where(np.array(classes) == intent)[<NUM_LIT:0>]] = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>curr[np.where(np.array(classes) == sample)[<NUM_LIT:0>]] = <NUM_LIT:1><EOL><DEDENT>y.append(curr)<EOL><DEDENT>y = np.asarray(y)<EOL>return y<EOL>

Convert labels to one-hot vectors for multi-class multi-label classification

Args:
    labels: list of samples where each sample is a class or a list of classes which sample belongs with
    classes: array of classes' names

Returns:
    2d array with one-hot representation of given samples


def proba2labels(proba: [list, np.ndarray], confident_threshold: float, classes:  [list, np.ndarray]) -> List[List]:

y = []<EOL>for sample in proba:<EOL><INDENT>to_add = np.where(sample > confident_threshold)[<NUM_LIT:0>]<EOL>if len(to_add) > <NUM_LIT:0>:<EOL><INDENT>y.append(np.array(classes)[to_add].tolist())<EOL><DEDENT>else:<EOL><INDENT>y.append(np.array([np.array(classes)[np.argmax(sample)]]).tolist())<EOL><DEDENT><DEDENT>return y<EOL>

Convert vectors of probabilities to labels using confident threshold
(if probability to belong with the class is bigger than confident_threshold, sample belongs with the class;
if no probabilities bigger than confident threshold, sample belongs with the class with the biggest probability)

Args:
    proba: list of samples where each sample is a vector of probabilities to belong with given classes
    confident_threshold (float): boundary of probability to belong with a class
    classes: array of classes' names

Returns:
    list of lists of labels for each sample


def proba2onehot(proba: [list, np.ndarray], confident_threshold: float, classes:  [list, np.ndarray]) -> np.ndarray:

return labels2onehot(proba2labels(proba, confident_threshold, classes), classes)<EOL>

Convert vectors of probabilities to one-hot representations using confident threshold

Args:
    proba: samples where each sample is a vector of probabilities to belong with given classes
    confident_threshold: boundary of probability to belong with a class
    classes: array of classes' names

Returns:
    2d array with one-hot representation of given samples


def __call__(self, q_vects: Union[csr_matrix, List]) -> Tuple[List[str], List[int]]:

if isinstance(q_vects[<NUM_LIT:0>], csr_matrix):<EOL><INDENT>q_norm = sparse_norm(q_vects)<EOL>if q_norm == <NUM_LIT:0.0>:<EOL><INDENT>cos_similarities = np.zeros((q_vects.shape[<NUM_LIT:0>], self.x_train_features.shape[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>norm = q_norm * sparse_norm(self.x_train_features, axis=<NUM_LIT:1>)<EOL>cos_similarities = np.array(q_vects.dot(self.x_train_features.T).todense())<EOL>cos_similarities = cos_similarities / norm<EOL><DEDENT><DEDENT>elif isinstance(q_vects[<NUM_LIT:0>], np.ndarray):<EOL><INDENT>q_vects = np.array(q_vects)<EOL>self.x_train_features = np.array(self.x_train_features)<EOL>norm = np.linalg.norm(q_vects) * np.linalg.norm(self.x_train_features, axis=<NUM_LIT:1>)<EOL>cos_similarities = q_vects.dot(self.x_train_features.T) / norm<EOL><DEDENT>elif q_vects[<NUM_LIT:0>] is None:<EOL><INDENT>cos_similarities = np.zeros(len(self.x_train_features))<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT>y_labels = np.unique(self.y_train)<EOL>labels_scores = np.zeros((len(cos_similarities), len(y_labels)))<EOL>for i, label in enumerate(y_labels):<EOL><INDENT>labels_scores[:, i] = np.max([cos_similarities[:, i]<EOL>for i, value in enumerate(self.y_train) if value == label], axis=<NUM_LIT:0>)<EOL><DEDENT>labels_scores_sum = labels_scores.sum(axis=<NUM_LIT:1>, keepdims=True)<EOL>labels_scores = np.divide(labels_scores, labels_scores_sum,<EOL>out=np.zeros_like(labels_scores), where=(labels_scores_sum != <NUM_LIT:0>))<EOL>answer_ids = np.argsort(labels_scores)[:, -self.top_n:]<EOL>answers = []<EOL>scores = []<EOL>for i in range(len(answer_ids)):<EOL><INDENT>answers.extend([y_labels[id] for id in answer_ids[i, ::-<NUM_LIT:1>]])<EOL>scores.extend([np.round(labels_scores[i, id], <NUM_LIT:2>) for id in answer_ids[i, ::-<NUM_LIT:1>]])<EOL><DEDENT>return answers, scores<EOL>

Found most similar answer for input vectorized question

        Parameters:
            q_vects: vectorized questions

        Returns:
            Tuple of Answer and Score


def fit(self, x_train_vects: Tuple[Union[csr_matrix, List]], y_train: Tuple[str]) -> None:

if isinstance(x_train_vects, tuple):<EOL><INDENT>if len(x_train_vects) != <NUM_LIT:0>:<EOL><INDENT>if isinstance(x_train_vects[<NUM_LIT:0>], csr_matrix):<EOL><INDENT>self.x_train_features = vstack(list(x_train_vects))<EOL><DEDENT>elif isinstance(x_train_vects[<NUM_LIT:0>], np.ndarray):<EOL><INDENT>self.x_train_features = np.vstack(list(x_train_vects))<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>self.x_train_features = x_train_vects<EOL><DEDENT>self.y_train = list(y_train)<EOL>

Train classifier

        Parameters:
            x_train_vects: vectorized question for train dataset
            y_train: answers for train dataset

        Returns:
            None


logger.info("<STR_LIT>".format(self.save_path))<EOL>save_pickle((self.x_train_features, self.y_train), self.save_path)<EOL>


logger.info("<STR_LIT>".format(self.load_path))<EOL>self.x_train_features, self.y_train = load_pickle(self.load_path)<EOL>


def train_on_batch(self, c_tokens: np.ndarray, c_chars: np.ndarray, q_tokens: np.ndarray, q_chars: np.ndarray,<EOL>y1s: Tuple[List[int], ...], y2s: Tuple[List[int], ...]) -> float:

<EOL>y1s = np.array([x[<NUM_LIT:0>] for x in y1s])<EOL>y2s = np.array([x[<NUM_LIT:0>] for x in y2s])<EOL>if self.noans_token:<EOL><INDENT>noans_mask = ((y1s != -<NUM_LIT:1>) * (y2s != -<NUM_LIT:1>))<EOL>y1s = (y1s + <NUM_LIT:1>) * noans_mask<EOL>y2s = (y2s + <NUM_LIT:1>) * noans_mask<EOL><DEDENT>feed_dict = self._build_feed_dict(c_tokens, c_chars, q_tokens, q_chars, y1s, y2s)<EOL>loss, _, lear_rate = self.sess.run([self.loss, self.train_op, self.lear_rate_ph],<EOL>feed_dict=feed_dict)<EOL>report = {'<STR_LIT>': loss, '<STR_LIT>': float(lear_rate), '<STR_LIT>': self.get_momentum()}<EOL>return report<EOL>

This method is called by trainer to make one training step on one batch.

Args:
    c_tokens: batch of tokenized contexts
    c_chars: batch of tokenized contexts, each token split on chars
    q_tokens: batch of tokenized questions
    q_chars: batch of tokenized questions, each token split on chars
    y1s: batch of ground truth answer start positions
    y2s: batch of ground truth answer end positions

Returns:
    value of loss function on batch


def __call__(self, c_tokens: np.ndarray, c_chars: np.ndarray, q_tokens: np.ndarray, q_chars: np.ndarray,<EOL>*args, **kwargs) -> Tuple[np.ndarray, np.ndarray, List[float]]:

if any(np.sum(c_tokens, axis=-<NUM_LIT:1>) == <NUM_LIT:0>) or any(np.sum(q_tokens, axis=-<NUM_LIT:1>) == <NUM_LIT:0>):<EOL><INDENT>logger.info('<STR_LIT>')<EOL>noanswers = -np.ones(shape=(c_tokens.shape[<NUM_LIT:0>]), dtype=np.int32)<EOL>zero_probs = np.zeros(shape=(c_tokens.shape[<NUM_LIT:0>]), dtype=np.float32)<EOL>if self.noans_token:<EOL><INDENT>return noanswers, noanswers, zero_probs, zero_probs<EOL><DEDENT>return noanswers, noanswers, zero_probs<EOL><DEDENT>feed_dict = self._build_feed_dict(c_tokens, c_chars, q_tokens, q_chars)<EOL>if self.noans_token:<EOL><INDENT>yp1, yp2, logits, score = self.sess.run([self.yp1, self.yp2, self.yp_logits, self.yp_score],<EOL>feed_dict=feed_dict)<EOL>noans_mask = (yp1 * yp2).astype(bool)<EOL>yp1 = yp1 * noans_mask - <NUM_LIT:1><EOL>yp2 = yp2 * noans_mask - <NUM_LIT:1><EOL>return yp1, yp2, logits.tolist(), score.tolist()<EOL><DEDENT>yp1, yp2, logits = self.sess.run([self.yp1, self.yp2, self.yp_logits], feed_dict=feed_dict)<EOL>return yp1, yp2, logits.tolist()<EOL>

Predicts answer start and end positions by given context and question.

Args:
    c_tokens: batch of tokenized contexts
    c_chars: batch of tokenized contexts, each token split on chars
    q_tokens: batch of tokenized questions
    q_chars: batch of tokenized questions, each token split on chars

Returns:
    answer_start, answer_end positions, answer logits which represent models confidence


def process_event(self, event_name: str, data) -> None:

super().process_event(event_name, data)<EOL>

Processes events sent by trainer. Implements learning rate decay.

Args:
    event_name: event_name sent by trainer
    data: number of examples, epochs, metrics sent by trainer


def dot_attention(inputs, memory, mask, att_size, keep_prob=<NUM_LIT:1.0>, scope="<STR_LIT>"):

with tf.variable_scope(scope):<EOL><INDENT>BS, IL, IH = tf.unstack(tf.shape(inputs))<EOL>BS, ML, MH = tf.unstack(tf.shape(memory))<EOL>d_inputs = tf.nn.dropout(inputs, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, IH])<EOL>d_memory = tf.nn.dropout(memory, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, MH])<EOL>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>inputs_att = tf.layers.dense(d_inputs, att_size, use_bias=False, activation=tf.nn.relu)<EOL>memory_att = tf.layers.dense(d_memory, att_size, use_bias=False, activation=tf.nn.relu)<EOL>logits = tf.matmul(inputs_att, tf.transpose(memory_att, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])) / (att_size ** <NUM_LIT:0.5>)<EOL>mask = tf.tile(tf.expand_dims(mask, axis=<NUM_LIT:1>), [<NUM_LIT:1>, IL, <NUM_LIT:1>])<EOL>att_weights = tf.nn.softmax(softmax_mask(logits, mask))<EOL>outputs = tf.matmul(att_weights, memory)<EOL>res = tf.concat([inputs, outputs], axis=<NUM_LIT:2>)<EOL><DEDENT>with tf.variable_scope("<STR_LIT>"):<EOL><INDENT>dim = res.get_shape().as_list()[-<NUM_LIT:1>]<EOL>d_res = tf.nn.dropout(res, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, IH + MH])<EOL>gate = tf.layers.dense(d_res, dim, use_bias=False, activation=tf.nn.sigmoid)<EOL>return res * gate<EOL><DEDENT><DEDENT>

Computes attention vector for each item in inputs:
       attention vector is a weighted sum of memory items.
       Dot product between input and memory vector is used as similarity measure.

       Gate mechanism is applied to attention vectors to produce output.

    Args:
        inputs: Tensor [batch_size x input_len x feature_size]
        memory: Tensor [batch_size x memory_len x feature_size]
        mask: inputs mask
        att_size: hidden size of attention
        keep_prob: dropout keep_prob
        scope:

    Returns:
        attention vectors [batch_size x input_len x (feature_size + feature_size)]


def simple_attention(memory, att_size, mask, keep_prob=<NUM_LIT:1.0>, scope="<STR_LIT>"):

with tf.variable_scope(scope):<EOL><INDENT>BS, ML, MH = tf.unstack(tf.shape(memory))<EOL>memory_do = tf.nn.dropout(memory, keep_prob=keep_prob, noise_shape=[BS, <NUM_LIT:1>, MH])<EOL>logits = tf.layers.dense(tf.layers.dense(memory_do, att_size, activation=tf.nn.tanh), <NUM_LIT:1>, use_bias=False)<EOL>logits = softmax_mask(tf.squeeze(logits, [<NUM_LIT:2>]), mask)<EOL>att_weights = tf.expand_dims(tf.nn.softmax(logits), axis=<NUM_LIT:2>)<EOL>res = tf.reduce_sum(att_weights * memory, axis=<NUM_LIT:1>)<EOL>return res<EOL><DEDENT>

Simple attention without any conditions.

       Computes weighted sum of memory elements.


def attention(inputs, state, att_size, mask, scope="<STR_LIT>"):

with tf.variable_scope(scope):<EOL><INDENT>u = tf.concat([tf.tile(tf.expand_dims(state, axis=<NUM_LIT:1>), [<NUM_LIT:1>, tf.shape(inputs)[<NUM_LIT:1>], <NUM_LIT:1>]), inputs], axis=<NUM_LIT:2>)<EOL>logits = tf.layers.dense(tf.layers.dense(u, att_size, activation=tf.nn.tanh), <NUM_LIT:1>, use_bias=False)<EOL>logits = softmax_mask(tf.squeeze(logits, [<NUM_LIT:2>]), mask)<EOL>att_weights = tf.expand_dims(tf.nn.softmax(logits), axis=<NUM_LIT:2>)<EOL>res = tf.reduce_sum(att_weights * inputs, axis=<NUM_LIT:1>)<EOL>return res, logits<EOL><DEDENT>

Computes weighted sum of inputs conditioned on state


elmo_module = hub.Module(self.spec, trainable=False)<EOL>sess_config = tf.ConfigProto()<EOL>sess_config.gpu_options.allow_growth = True<EOL>sess = tf.Session(config=sess_config)<EOL>tokens_ph = tf.placeholder(shape=(None, None), dtype=tf.string, name='<STR_LIT>')<EOL>tokens_length_ph = tf.placeholder(shape=(None,), dtype=tf.int32, name='<STR_LIT>')<EOL>elmo_outputs = elmo_module(inputs={"<STR_LIT>": tokens_ph,<EOL>"<STR_LIT>": tokens_length_ph},<EOL>signature="<STR_LIT>",<EOL>as_dict=True)<EOL>sess.run(tf.global_variables_initializer())<EOL>return elmo_outputs, sess, tokens_ph, tokens_length_ph<EOL>

Load a ELMo TensorFlow Hub Module from a self.spec.

Returns:
    ELMo pre-trained model wrapped in TenserFlow Hub Module.


if not batch:<EOL><INDENT>empty_vec = np.zeros(self.dim, dtype=np.float32)<EOL>return [empty_vec] if '<STR_LIT:default>' in self.elmo_output_names else [[empty_vec]]<EOL><DEDENT>filled_batch = []<EOL>for batch_line in batch:<EOL><INDENT>batch_line = batch_line if batch_line else ['<STR_LIT>']<EOL>filled_batch.append(batch_line)<EOL><DEDENT>batch = filled_batch<EOL>if self.max_token:<EOL><INDENT>batch = [batch_line[:self.max_token] for batch_line in batch]<EOL><DEDENT>tokens_length = [len(batch_line) for batch_line in batch]<EOL>tokens_length_max = max(tokens_length)<EOL>batch = [batch_line + ['<STR_LIT>']*(tokens_length_max - len(batch_line)) for batch_line in batch]<EOL>return batch, tokens_length<EOL>

Fill batch correct values.

Args:
    batch: A list of tokenized text samples.

Returns:
    batch: A list of tokenized text samples.


def _mini_batch_fit(self, batch: List[List[str]], *args, **kwargs) -> Union[List[np.ndarray], np.ndarray]:

batch, tokens_length = self._fill_batch(batch)<EOL>elmo_outputs = self.sess.run(self.elmo_outputs,<EOL>feed_dict={self.tokens_ph: batch,<EOL>self.tokens_length_ph: tokens_length})<EOL>if '<STR_LIT:default>' in self.elmo_output_names:<EOL><INDENT>elmo_output_values = elmo_outputs['<STR_LIT:default>']<EOL>dim0, dim1 = elmo_output_values.shape<EOL>if self.dim != dim1:<EOL><INDENT>shape = (dim0, self.dim if isinstance(self.dim, int) else self.dim[<NUM_LIT:0>])<EOL>elmo_output_values = np.resize(elmo_output_values, shape)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>elmo_output_values = [elmo_outputs[elmo_output_name] for elmo_output_name in self.elmo_output_names]<EOL>elmo_output_values = np.concatenate(elmo_output_values, axis=-<NUM_LIT:1>)<EOL>dim0, dim1, dim2 = elmo_output_values.shape<EOL>if self.concat_last_axis and self.dim != dim2:<EOL><INDENT>shape = (dim0, dim1, self.dim)<EOL>elmo_output_values = np.resize(elmo_output_values, shape)<EOL><DEDENT>elmo_output_values = [elmo_output_values_line[:length_line]<EOL>for length_line, elmo_output_values_line in zip(tokens_length, elmo_output_values)]<EOL>if not self.concat_last_axis:<EOL><INDENT>slice_indexes = np.cumsum(self.dim).tolist()[:-<NUM_LIT:1>]<EOL>elmo_output_values = [[np.array_split(vec, slice_indexes) for vec in tokens]<EOL>for tokens in elmo_output_values]<EOL><DEDENT><DEDENT>return elmo_output_values<EOL>

Embed sentences from a batch.

Args:
    batch: A list of tokenized text samples.

Returns:
    A batch of ELMo embeddings.


@overrides<EOL><INDENT>def __call__(self, batch: List[List[str]],<EOL>*args, **kwargs) -> Union[List[np.ndarray], np.ndarray]:<DEDENT>

if len(batch) > self.mini_batch_size:<EOL><INDENT>batch_gen = chunk_generator(batch, self.mini_batch_size)<EOL>elmo_output_values = []<EOL>for mini_batch in batch_gen:<EOL><INDENT>mini_batch_out = self._mini_batch_fit(mini_batch, *args, **kwargs)<EOL>elmo_output_values.extend(mini_batch_out)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>elmo_output_values = self._mini_batch_fit(batch, *args, **kwargs)<EOL><DEDENT>if self.pad_zero:<EOL><INDENT>elmo_output_values = zero_pad(elmo_output_values)<EOL><DEDENT>return elmo_output_values<EOL>


yield from ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>

Iterate over all words from a ELMo model vocabulary.
The ELMo model vocabulary consists of ``['<S>', '</S>', '<UNK>']``.

Returns:
    An iterator of three elements ``['<S>', '</S>', '<UNK>']``.


@staticmethod<EOL><INDENT>def load_tags_vocab(load_path: str) -> dict:<DEDENT>

tags_vocab = dict()<EOL>with open(load_path, '<STR_LIT:r>') as f:<EOL><INDENT>lines = f.readlines()<EOL>f.close()<EOL><DEDENT>for line in lines:<EOL><INDENT>key, val = line[:-<NUM_LIT:1>].split('<STR_LIT:U+0020>')  <EOL>tags_vocab[key] = val<EOL><DEDENT>return tags_vocab<EOL>

Load tag vocabulary from the given path, each key of the vocabulary is a tag, \
    and the corresponding value of the item is a coefficient of words with such tags to be multiplied for.

Args:
    load_path: path to the vocabulary to be load from

Returns:
    vocabulary


@staticmethod<EOL><INDENT>def load_counter_vocab(load_path: str) -> Tuple[dict, int]:<DEDENT>

counter_vocab = dict()<EOL>with open(load_path, '<STR_LIT:r>') as f:<EOL><INDENT>lines = f.readlines()<EOL>f.close()<EOL><DEDENT>min_val = np.inf<EOL>for line in lines:<EOL><INDENT>key, val = line[:-<NUM_LIT:1>].split('<STR_LIT:\t>')<EOL>val = int(val)<EOL>counter_vocab[key] = val<EOL>if val < min_val:<EOL><INDENT>min_val = val<EOL><DEDENT><DEDENT>return counter_vocab, min_val<EOL>

Load counter vocabulary from the given path

Args:
    load_path: path to the vocabulary to be load from

Returns:
    vocabulary


@staticmethod<EOL><INDENT>def space_detokenizer(batch: List[List[str]]) -> List[str]:<DEDENT>

return ["<STR_LIT:U+0020>".join(tokens) for tokens in batch]<EOL>

Detokenizer by default. Linking tokens by space symbol

Args:
    batch: batch of tokenized texts

Returns:
    batch of detokenized texts


@overrides<EOL><INDENT>def __call__(self, batch: List[List[str]], tags_batch: Optional[List[List[str]]] = None, mean: bool = None,<EOL>*args, **kwargs) -> List[Union[list, np.ndarray]]:<DEDENT>

if self.tags_vocab:<EOL><INDENT>if tags_batch is None:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>batch = [self._tags_encode(sample, tags_sample, mean=mean) for sample, tags_sample in zip(batch, tags_batch)]<EOL><DEDENT>else:<EOL><INDENT>if tags_batch:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>batch = [self._encode(sample, mean=mean) for sample in batch]<EOL><DEDENT>if self.pad_zero:<EOL><INDENT>batch = zero_pad(batch)<EOL><DEDENT>return batch<EOL>

Infer on the given data

Args:
    batch: tokenized text samples
    tags_batch: optional batch of corresponding tags
    mean: whether to return mean token embedding (does not depend on self.mean)
    *args: additional arguments
    **kwargs: additional arguments

Returns:


def _encode(self, tokens: List[str], mean: bool) -> Union[List[np.ndarray], np.ndarray]:

if self.vectorizer:<EOL><INDENT>detokenized_sample = self.tokenizer([tokens])[<NUM_LIT:0>]  <EOL>vectorized_sample = self.vectorizer([detokenized_sample])  <EOL>weights = np.array([vectorized_sample[<NUM_LIT:0>, np.where(self.vocabulary == token)[<NUM_LIT:0>][<NUM_LIT:0>]]<EOL>if len(np.where(self.vocabulary == token)[<NUM_LIT:0>]) else <NUM_LIT:0.><EOL>for token in tokens])<EOL><DEDENT>else:<EOL><INDENT>weights = np.array([self.get_weight(max(self.counter_vocab.get(token, <NUM_LIT:0>), self.idf_base_count))<EOL>for token in tokens])<EOL><DEDENT>if sum(weights) == <NUM_LIT:0>:<EOL><INDENT>weights = np.ones(len(tokens))<EOL><DEDENT>embedded_tokens = np.array(self.embedder([tokens]))[<NUM_LIT:0>, :, :]<EOL>if mean is None:<EOL><INDENT>mean = self.mean<EOL><DEDENT>if mean:<EOL><INDENT>embedded_tokens = np.average(embedded_tokens, weights=weights, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>embedded_tokens = np.array([weights[i] * embedded_tokens[i] for i in range(len(tokens))])<EOL><DEDENT>return embedded_tokens<EOL>

Embed one text sample

Args:
    tokens: tokenized text sample
    mean: whether to return mean token embedding (does not depend on self.mean)

Returns:
    list of embedded tokens or array of mean values


def get_weight(self, count: int) -> float:

log_count = np.log(count) / np.log(self.log_base)<EOL>log_base_count = np.log(self.idf_base_count) / np.log(self.log_base)<EOL>weight = max(<NUM_LIT:1.0> / (<NUM_LIT:1.0> + log_count - log_base_count), self.min_idf_weight)<EOL>return weight<EOL>

Calculate the weight corresponding to the given count

Args:
    count: the number of occurences of particular token

Returns:
    weight


def _tags_encode(self, tokens: List[str], tags: List[str], mean: bool) -> Union[List[np.ndarray], np.ndarray]:

embedded_tokens = np.array(self.embedder([tokens]))[<NUM_LIT:0>, :, :]<EOL>tags_weights = np.array([self.tags_vocab.get(tag, <NUM_LIT:1.0>) for tag in tags])<EOL>detokenized_sample = self.tokenizer([tokens])[<NUM_LIT:0>]  <EOL>vectorized_sample = self.vectorizer([detokenized_sample])  <EOL>if self.vectorizer:<EOL><INDENT>weights = np.array([vectorized_sample[<NUM_LIT:0>, np.where(self.vocabulary == token)[<NUM_LIT:0>][<NUM_LIT:0>]]<EOL>if len(np.where(self.vocabulary == token)[<NUM_LIT:0>]) else <NUM_LIT:0.><EOL>for token in tokens])<EOL><DEDENT>else:<EOL><INDENT>weights = np.array([self.get_weight(max(self.counter_vocab.get(token, <NUM_LIT:0>), self.idf_base_count))<EOL>for token in tokens])<EOL><DEDENT>weights = np.multiply(weights, tags_weights)<EOL>if sum(weights) == <NUM_LIT:0>:<EOL><INDENT>weights = np.ones(len(tokens))<EOL><DEDENT>if mean is None:<EOL><INDENT>mean = self.mean<EOL><DEDENT>if mean:<EOL><INDENT>embedded_tokens = np.average(embedded_tokens, weights=weights, axis=<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>embedded_tokens = np.array([weights[i] * embedded_tokens[i] for i in range(len(tokens))])<EOL><DEDENT>return embedded_tokens<EOL>

Embed one text sample

Args:
    tokens: tokenized text sample
    tags: tokenized tags sample
    mean: whether to return mean token embedding (does not depend on self.mean)

Returns:
    list of embedded tokens or array of mean values


log.info(f"<STR_LIT>")<EOL>if not self.load_path.exists():<EOL><INDENT>log.warning(f'<STR_LIT>')<EOL>return<EOL><DEDENT>self.model = KeyedVectors.load_word2vec_format(str(self.load_path))<EOL>self.dim = self.model.vector_size<EOL>


@overrides<EOL><INDENT>def __iter__(self) -> Iterator[str]:<DEDENT>

yield from self.model.vocab<EOL>

Iterate over all words from GloVe model vocabulary

Returns:
    iterator


log.info(f"<STR_LIT>")<EOL>self.model = fastText.load_model(str(self.load_path))<EOL>self.dim = self.model.get_dimension()<EOL>

Load fastText binary model from self.load_path


yield from self.model.get_words()<EOL>

Iterate over all words from fastText model vocabulary

Returns:
    iterator


def __init__(self, load_path: Union[str, Path], pad_zero: bool = False, mean: bool = False, **kwargs) -> None:

super().__init__(save_path=None, load_path=load_path)<EOL>self.tok2emb = {}<EOL>self.pad_zero = pad_zero<EOL>self.mean = mean<EOL>self.dim = None<EOL>self.model = None<EOL>self.load()<EOL>

Initialize embedder with given parameters


@overrides<EOL><INDENT>def save(self) -> None:<DEDENT>

raise NotImplementedError<EOL>

Class does not save loaded model again as it is not trained during usage


@overrides<EOL><INDENT>def __call__(self, batch: List[List[str]], mean: bool = None) -> List[Union[list, np.ndarray]]:<DEDENT>

batch = [self._encode(sample, mean) for sample in batch]<EOL>if self.pad_zero:<EOL><INDENT>batch = zero_pad(batch)<EOL><DEDENT>return batch<EOL>

Embed sentences from batch

Args:
    batch: list of tokenized text samples
    mean: whether to return mean embedding of tokens per sample

Returns:
    embedded batch


@abstractmethod<EOL><INDENT>def __iter__(self) -> Iterator[str]:<DEDENT>

Iterate over all words from the model vocabulary

Returns:
    iterator


@abstractmethod<EOL><INDENT>def _get_word_vector(self, w: str) -> np.ndarray:<DEDENT>

Embed a word using ``self.model``

Args:
    w: a word

Returns:
    embedding vector


embedded_tokens = []<EOL>for t in tokens:<EOL><INDENT>try:<EOL><INDENT>emb = self.tok2emb[t]<EOL><DEDENT>except KeyError:<EOL><INDENT>try:<EOL><INDENT>emb = self._get_word_vector(t)<EOL><DEDENT>except KeyError:<EOL><INDENT>emb = np.zeros(self.dim, dtype=np.float32)<EOL><DEDENT>self.tok2emb[t] = emb<EOL><DEDENT>embedded_tokens.append(emb)<EOL><DEDENT>if mean is None:<EOL><INDENT>mean = self.mean<EOL><DEDENT>if mean:<EOL><INDENT>filtered = [et for et in embedded_tokens if np.any(et)]<EOL>if filtered:<EOL><INDENT>return np.mean(filtered, axis=<NUM_LIT:0>)<EOL><DEDENT>return np.zeros(self.dim, dtype=np.float32)<EOL><DEDENT>return embedded_tokens<EOL>

Embed one text sample

Args:
    tokens: tokenized text sample
    mean: whether to return mean embedding of tokens per sample

Returns:
    list of embedded tokens or array of mean values


def __call__(self, batch: Union[List[str], List[List[str]]]) ->Union[List[List[str]], List[str]]:

if isinstance(batch[<NUM_LIT:0>], str):<EOL><INDENT>if self.lemmas:<EOL><INDENT>return list(self._lemmatize(batch))<EOL><DEDENT>else:<EOL><INDENT>return list(self._tokenize(batch))<EOL><DEDENT><DEDENT>if isinstance(batch[<NUM_LIT:0>], list):<EOL><INDENT>return [detokenize(doc) for doc in batch]<EOL><DEDENT>raise TypeError(<EOL>"<STR_LIT>".format(type(batch[<NUM_LIT:0>])))<EOL>

Tokenize or detokenize strings, depends on the type structure of passed arguments.

        Args:
            batch: a batch of documents to perform tokenizing/lemmatizing;
             or a batch of lists of tokens/lemmas to perform detokenizing

        Returns:
            a batch of lists of tokens/lemmas; or a batch of detokenized strings

        Raises:
            TypeError: If the first element of ``batch`` is neither ``List``, nor ``str``.


def _tokenize(self, data: List[str], ngram_range: Tuple[int, int]=(<NUM_LIT:1>, <NUM_LIT:1>), lowercase: bool=True)-> Generator[List[str], Any, None]:

<EOL>_ngram_range = self.ngram_range or ngram_range<EOL>if self.lowercase is None:<EOL><INDENT>_lowercase = lowercase<EOL><DEDENT>else:<EOL><INDENT>_lowercase = self.lowercase<EOL><DEDENT>for i, doc in enumerate(data):<EOL><INDENT>tokens = self.tokenizer.tokenize(doc)<EOL>if _lowercase:<EOL><INDENT>tokens = [t.lower() for t in tokens]<EOL><DEDENT>filtered = self._filter(tokens)<EOL>processed_doc = ngramize(filtered, ngram_range=_ngram_range)<EOL>yield from processed_doc<EOL><DEDENT>

Tokenize a list of documents.

       Args:
           data: a list of documents to tokenize
           ngram_range: size of ngrams to create; only unigrams are returned by default
           lowercase: whether to perform lowercasing or not; is performed by default by
           :meth:`_tokenize` and :meth:`_lemmatize` methods

       Yields:
           list of lists of ngramized tokens or list of detokenized strings

        Returns:
            None


def _lemmatize(self, data: List[str], ngram_range: Tuple[int, int]=(<NUM_LIT:1>, <NUM_LIT:1>)) ->Generator[List[str], Any, None]:

<EOL>_ngram_range = self.ngram_range or ngram_range<EOL>tokenized_data = list(self._tokenize(data))<EOL>for i, doc in enumerate(tokenized_data):<EOL><INDENT>lemmas = []<EOL>for token in doc:<EOL><INDENT>try:<EOL><INDENT>lemma = self.tok2morph[token]<EOL><DEDENT>except KeyError:<EOL><INDENT>lemma = self.lemmatizer.parse(token)[<NUM_LIT:0>].normal_form<EOL>self.tok2morph[token] = lemma<EOL><DEDENT>lemmas.append(lemma)<EOL><DEDENT>filtered = self._filter(lemmas)<EOL>processed_doc = ngramize(filtered, ngram_range=_ngram_range)<EOL>yield from processed_doc<EOL><DEDENT>

Lemmatize a list of documents.

        Args:
            data: a list of documents to tokenize
            ngram_range: size of ngrams to create; only unigrams are returned by default

        Yields:
            list of lists of ngramized tokens or list of detokenized strings

        Returns:
            None


def _filter(self, items: List[str], alphas_only: bool=True) -> List[str]:

if self.alphas_only is None:<EOL><INDENT>_alphas_only = alphas_only<EOL><DEDENT>else:<EOL><INDENT>_alphas_only = self.alphas_only<EOL><DEDENT>if _alphas_only:<EOL><INDENT>filter_fn = lambda x: x.isalpha() and not x.isspace() and x not in self.stopwords<EOL><DEDENT>else:<EOL><INDENT>filter_fn = lambda x: not x.isspace() and x not in self.stopwords<EOL><DEDENT>return list(filter(filter_fn, items))<EOL>

Filter a list of tokens/lemmas.

        Args:
            items: a list of tokens/lemmas to filter
            alphas_only: whether to filter out non-alpha tokens

        Returns:
            a list of filtered tokens/lemmas


def set_stopwords(self, stopwords: List[str]) -> None:

self.stopwords = stopwords<EOL>

Redefine a list of stopwords.

       Args:
           stopwords: a list of stopwords

       Returns:
           None


def __call__(self, batch: List[str]) -> List[List[str]]:

if isinstance(batch, (list, tuple)):<EOL><INDENT>return [sample.split() for sample in batch]<EOL><DEDENT>else:<EOL><INDENT>raise NotImplementedError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>

Tokenize given batch

Args:
    batch: list of texts to tokenize

Returns:
    tokenized batch


Tokenize or detokenize strings, depends on the type structure of passed arguments.

        Args:
            batch: a batch of documents to perform tokenizing/lemmatizing;
             or a batch of lists of tokens/lemmas to perform detokenizing

        Returns:
            a batch of lists of tokens/lemmas; or a batch of detokenized strings

        Raises:
            TypeError: If the first element of ``batch`` is neither List, nor str.


def _tokenize(self, data: List[str], ngram_range: Tuple[int, int]=(<NUM_LIT:1>, <NUM_LIT:1>), batch_size: int=<NUM_LIT>,<EOL>n_threads: int=<NUM_LIT:1>, lowercase: bool=True) -> Generator[List[str], Any, None]:

<EOL>_batch_size = self.batch_size or batch_size<EOL>_ngram_range = self.ngram_range or ngram_range<EOL>_n_threads = self.n_threads or n_threads<EOL>if self.lowercase is None:<EOL><INDENT>_lowercase = lowercase<EOL><DEDENT>else:<EOL><INDENT>_lowercase = self.lowercase<EOL><DEDENT>for i, doc in enumerate(<EOL>self.tokenizer.pipe(data, batch_size=_batch_size, n_threads=_n_threads)):<EOL><INDENT>if _lowercase:<EOL><INDENT>tokens = [t.lower_ for t in doc]<EOL><DEDENT>else:<EOL><INDENT>tokens = [t.text for t in doc]<EOL><DEDENT>filtered = self._filter(tokens)<EOL>processed_doc = ngramize(filtered, ngram_range=_ngram_range)<EOL>yield from processed_doc<EOL><DEDENT>

Tokenize a list of documents.

        Args:
            data: a list of documents to tokenize
            ngram_range: size of ngrams to create; only unigrams are returned by default
            batch_size: a batch size for inner spacy multi-threading
            n_threads: a number of threads for inner spacy multi-threading
            lowercase: whether to perform lowercasing or not; is performed by default by
             :meth:`_tokenize` and :meth:`_lemmatize` methods

        Yields:
            list of lists of ngramized tokens or list of detokenized strings

        Returns:
            None


def _lemmatize(self, data: List[str], ngram_range: Tuple[int, int]=(<NUM_LIT:1>, <NUM_LIT:1>), batch_size: int=<NUM_LIT>,<EOL>n_threads: int=<NUM_LIT:1>) -> Generator[List[str], Any, None]:

<EOL>_batch_size = self.batch_size or batch_size<EOL>_ngram_range = self.ngram_range or ngram_range<EOL>_n_threads = self.n_threads or n_threads<EOL>for i, doc in enumerate(<EOL>self.model.pipe(data, batch_size=_batch_size, n_threads=_n_threads)):<EOL><INDENT>lemmas = chain.from_iterable([sent.lemma_.split() for sent in doc.sents])<EOL>filtered = self._filter(lemmas)<EOL>processed_doc = ngramize(filtered, ngram_range=_ngram_range)<EOL>yield from processed_doc<EOL><DEDENT>

Lemmatize a list of documents.

        Args:
            data: a list of documents to tokenize
            ngram_range: size of ngrams to create; only unigrams are returned by default
            batch_size: a batch size for inner spacy multi-threading
            n_threads: a number of threads for inner spacy multi-threading

       Yields:
           list of lists of ngramized lemmas or list of detokenized strings

        Returns:
            None