def attention_bah_step(hidden_for_sketch, hidden_for_attn_alignment, sketch, cum_att):

with tf.name_scope('<STR_LIT>'):<EOL><INDENT>sketch_dims = hidden_for_sketch.get_shape().as_list()<EOL>batch_size = sketch_dims[<NUM_LIT:0>]<EOL>num_tokens = sketch_dims[<NUM_LIT:1>]<EOL>hidden_size = sketch_dims[<NUM_LIT:2>]<EOL>attn_alignment_dims = hidden_for_attn_alignment.get_shape().as_list()<EOL>attn_alignment_hidden_size = attn_alignment_dims[<NUM_LIT:2>]<EOL>repeated_sketch = tf.tile(tf.reshape(sketch, [-<NUM_LIT:1>, <NUM_LIT:1>, hidden_size]), (<NUM_LIT:1>,num_tokens, <NUM_LIT:1>))<EOL>concat_mem = tf.concat([hidden_for_sketch, repeated_sketch],-<NUM_LIT:1>)<EOL>concat_mem = tf.reshape(concat_mem, [-<NUM_LIT:1>, num_tokens, <NUM_LIT:2>*hidden_size]) <EOL>reduce_mem = tf.layers.dense(concat_mem, hidden_size)<EOL>score = tf.squeeze(tf.layers.dense(reduce_mem, units = <NUM_LIT:1>,<EOL>use_bias=False),-<NUM_LIT:1>)<EOL>inv_cum_att = tf.reshape(tf.ones_like(cum_att) - cum_att, [-<NUM_LIT:1>, num_tokens])<EOL>att = csoftmax(score, inv_cum_att)<EOL>t_reduce_mem = tf.transpose(reduce_mem, [<NUM_LIT:0>,<NUM_LIT:2>,<NUM_LIT:1>])<EOL>t_hidden_for_attn_alignment = tf.transpose(hidden_for_attn_alignment, [<NUM_LIT:0>,<NUM_LIT:2>,<NUM_LIT:1>])<EOL>r_att = tf.reshape(att, [-<NUM_LIT:1>, num_tokens, <NUM_LIT:1>])<EOL>next_sketch = tf.squeeze(tf.matmul(t_reduce_mem,r_att),-<NUM_LIT:1>)<EOL>aligned_hidden_sketch = tf.squeeze(tf.matmul(t_hidden_for_attn_alignment,r_att),-<NUM_LIT:1>)<EOL><DEDENT>return next_sketch, att, aligned_hidden_sketch<EOL>

It is a implementation one step of block of the Bahdanau et al. attention mechanism with concat score and the constrained softmax (csoftmax).
        Based on the papers:
            https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
            https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        sketch: A previous step sketch tensor for a sketch computing. This tensor have dimensionality [None, sketch_hidden_size]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        cum_att: A cumulative attention tensor with dimensionality [None, max_num_tokens]
    Returns:
        next_sketch: Tensor of the current step sketch with dimensionality [None, sketch_hidden_size]
        att: Tensor of the current step attention with dimensionality [None, max_num_tokens]
        aligned_hidden_sketch: Tensor of aligned hidden state of current step with dimensionality [None, hidden_size_for_attn_alignment]


def attention_bah_block(hidden_for_sketch, hidden_for_attn_alignment, attention_depth):

with tf.name_scope('<STR_LIT>'):<EOL><INDENT>sketch_dims = tf.shape(hidden_for_sketch)<EOL>batch_size = sketch_dims[<NUM_LIT:0>]<EOL>num_tokens = sketch_dims[<NUM_LIT:1>]<EOL>hidden_size = sketch_dims[<NUM_LIT:2>]<EOL>attn_alignment_dims = tf.shape(hidden_for_attn_alignment)<EOL>attn_alignment_hidden_size = attn_alignment_dims[<NUM_LIT:2>]<EOL>sketches = [tf.zeros(shape=[batch_size, hidden_size], dtype=tf.float32)]<EOL>aligned_hiddens = []<EOL>cum_att = tf.zeros(shape=[batch_size, num_tokens])  <EOL>for i in range(attention_depth):<EOL><INDENT>sketch, cum_att_, aligned_hidden = attention_bah_step(hidden_for_sketch, hidden_for_attn_alignment, sketches[-<NUM_LIT:1>], cum_att)<EOL>sketches.append(sketch) <EOL>aligned_hiddens.append(aligned_hidden) <EOL>cum_att += cum_att_<EOL><DEDENT>final_aligned_hiddens = tf.reshape(tf.transpose(tf.stack(aligned_hiddens), [<NUM_LIT:1>, <NUM_LIT:0>, <NUM_LIT:2>]),[<NUM_LIT:1>, attention_depth, attn_alignment_hidden_size])<EOL><DEDENT>return final_aligned_hiddens<EOL>

It is a implementation of the Bahdanau et al. attention mechanism with concat score and the constrained softmax (csoftmax).
        Based on the papers:
            https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
            https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        hidden_for_sketch: A tensorflow tensor for a sketch computing. This tensor have dimensionality [None, max_num_tokens, sketch_hidden_size]
        hidden_for_attn_alignment: A tensorflow tensor is aligned for output during a performing. This tensor have dimensionality [None, max_num_tokens, hidden_size_for_attn_alignment]
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        attention_depth: Number of usage csoftmax
    Returns:
        final_aligned_hiddens: Tensor at the output with dimensionality [1, attention_depth, hidden_size_for_attn_alignment]


def general_attention(key, context, hidden_size, projected_align=False):

if hidden_size % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_key =tf.layers.dense(key, hidden_size, kernel_initializer=xav())<EOL>r_projected_key = tf.reshape(projected_key, shape=[-<NUM_LIT:1>, hidden_size, <NUM_LIT:1>])<EOL>lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>(output_fw, output_bw), states =tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,<EOL>cell_bw=lstm_bw_cell,<EOL>inputs=r_context,<EOL>dtype=tf.float32)<EOL>bilstm_output = tf.concat([output_fw, output_bw], -<NUM_LIT:1>)<EOL>attn = tf.nn.softmax(tf.matmul(bilstm_output, r_projected_key), dim=<NUM_LIT:1>)<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(bilstm_output, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(r_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Luong et al. attention mechanism with general score. Based on the paper:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]


def light_general_attention(key, context, hidden_size, projected_align=False):

batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())<EOL>r_projected_key = tf.reshape(projected_key, shape=[-<NUM_LIT:1>, hidden_size, <NUM_LIT:1>])<EOL>projected_context =tf.layers.dense(r_context, hidden_size, kernel_initializer=xav())<EOL>attn = tf.nn.softmax(tf.matmul(projected_context, r_projected_key), dim=<NUM_LIT:1>)<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(projected_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(r_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Luong et al. attention mechanism with general score. Based on the paper:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]


def cs_general_attention(key, context, hidden_size, depth, projected_align=False):

if hidden_size % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>key_size = tf.shape(key)[-<NUM_LIT:1>]<EOL>batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_context = tf.layers.dense(r_context, token_size,<EOL>kernel_initializer=xav(),<EOL>name='<STR_LIT>')<EOL>lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>(output_fw, output_bw), states =tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,<EOL>cell_bw=lstm_bw_cell,<EOL>inputs=projected_context,<EOL>dtype=tf.float32)<EOL>bilstm_output = tf.concat([output_fw, output_bw], -<NUM_LIT:1>)<EOL>h_state_for_sketch = bilstm_output<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>h_state_for_attn_alignment = bilstm_output<EOL>aligned_h_state = csoftmax_attention.attention_gen_block(<EOL>h_state_for_sketch, h_state_for_attn_alignment, key, depth)<EOL>output =tf.reshape(aligned_h_state, shape=[batch_size, -<NUM_LIT:1>, depth * hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>h_state_for_attn_alignment = projected_context<EOL>aligned_h_state = csoftmax_attention.attention_gen_block(<EOL>h_state_for_sketch, h_state_for_attn_alignment, key, depth)<EOL>output =tf.reshape(aligned_h_state, shape=[batch_size, -<NUM_LIT:1>, depth * token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Luong et al. attention mechanism with general score and the constrained softmax (csoftmax).
        Based on the papers:
        https://arxiv.org/abs/1508.04025 "Effective Approaches to Attention-based Neural Machine Translation"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]


def bahdanau_attention(key, context, hidden_size, projected_align=False):

if hidden_size % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())<EOL>r_projected_key =tf.tile(tf.reshape(projected_key, shape=[-<NUM_LIT:1>, <NUM_LIT:1>, hidden_size]),<EOL>[<NUM_LIT:1>, max_num_tokens, <NUM_LIT:1>])<EOL>lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>(output_fw, output_bw), states =tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,<EOL>cell_bw=lstm_bw_cell,<EOL>inputs=r_context,<EOL>dtype=tf.float32)<EOL>bilstm_output = tf.concat([output_fw, output_bw], -<NUM_LIT:1>)<EOL>concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -<NUM_LIT:1>)<EOL>projected_state =tf.layers.dense(concat_h_state, hidden_size, use_bias=False,<EOL>kernel_initializer=xav())<EOL>score =tf.layers.dense(tf.tanh(projected_state), units=<NUM_LIT:1>, use_bias=False,<EOL>kernel_initializer=xav())<EOL>attn = tf.nn.softmax(score, dim=<NUM_LIT:1>)<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(bilstm_output, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(r_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]


def light_bahdanau_attention(key, context, hidden_size, projected_align=False):

batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav())<EOL>r_projected_key =tf.tile(tf.reshape(projected_key, shape=[-<NUM_LIT:1>, <NUM_LIT:1>, hidden_size]),<EOL>[<NUM_LIT:1>, max_num_tokens, <NUM_LIT:1>])<EOL>projected_context =tf.layers.dense(r_context, hidden_size, kernel_initializer=xav())<EOL>concat_h_state = tf.concat([projected_context, r_projected_key], -<NUM_LIT:1>)<EOL>projected_state =tf.layers.dense(concat_h_state, hidden_size, use_bias=False,<EOL>kernel_initializer=xav())<EOL>score =tf.layers.dense(tf.tanh(projected_state), units=<NUM_LIT:1>, use_bias=False,<EOL>kernel_initializer=xav())<EOL>attn = tf.nn.softmax(score, dim=<NUM_LIT:1>)<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(projected_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>t_context = tf.transpose(r_context, [<NUM_LIT:0>, <NUM_LIT:2>, <NUM_LIT:1>])<EOL>output = tf.reshape(tf.matmul(t_context, attn),<EOL>shape=[batch_size, -<NUM_LIT:1>, token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Bahdanau et al. attention mechanism. Based on the paper:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        projected_align: Using dense layer for hidden representation of context.
        If true, between input and attention mechanism insert a dense layer with dimensionality [hidden_size].
        If false, a dense layer is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, hidden_size]


def cs_bahdanau_attention(key, context, hidden_size, depth, projected_align=False):

if hidden_size % <NUM_LIT:2> != <NUM_LIT:0>:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>batch_size = tf.shape(context)[<NUM_LIT:0>]<EOL>max_num_tokens, token_size = context.get_shape().as_list()[-<NUM_LIT:2>:]<EOL>r_context = tf.reshape(context, shape=[-<NUM_LIT:1>, max_num_tokens, token_size])<EOL>projected_context = tf.layers.dense(r_context, token_size,<EOL>kernel_initializer=xav(),<EOL>name='<STR_LIT>')<EOL>projected_key = tf.layers.dense(key, hidden_size, kernel_initializer=xav(),<EOL>name='<STR_LIT>')<EOL>r_projected_key =tf.tile(tf.reshape(projected_key, shape=[-<NUM_LIT:1>, <NUM_LIT:1>, hidden_size]),<EOL>[<NUM_LIT:1>, max_num_tokens, <NUM_LIT:1>])<EOL>lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(hidden_size//<NUM_LIT:2>)<EOL>(output_fw, output_bw), states =tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,<EOL>cell_bw=lstm_bw_cell,<EOL>inputs=projected_context,<EOL>dtype=tf.float32)<EOL>bilstm_output = tf.concat([output_fw, output_bw], -<NUM_LIT:1>)<EOL>concat_h_state = tf.concat([r_projected_key, output_fw, output_bw], -<NUM_LIT:1>)<EOL>if projected_align:<EOL><INDENT>log.info("<STR_LIT>")<EOL>h_state_for_attn_alignment = bilstm_output<EOL>aligned_h_state = csoftmax_attention.attention_bah_block(<EOL>concat_h_state, h_state_for_attn_alignment, depth)<EOL>output =tf.reshape(aligned_h_state, shape=[batch_size, -<NUM_LIT:1>, depth * hidden_size])<EOL><DEDENT>else:<EOL><INDENT>log.info("<STR_LIT>")<EOL>h_state_for_attn_alignment = projected_context<EOL>aligned_h_state = csoftmax_attention.attention_bah_block(<EOL>concat_h_state, h_state_for_attn_alignment, depth)<EOL>output =tf.reshape(aligned_h_state, shape=[batch_size, -<NUM_LIT:1>, depth * token_size])<EOL><DEDENT>return output<EOL>

It is a implementation of the Bahdanau et al. attention mechanism. Based on the papers:
        https://arxiv.org/abs/1409.0473 "Neural Machine Translation by Jointly Learning to Align and Translate"
        https://andre-martins.github.io/docs/emnlp2017_final.pdf "Learning What's Easy: Fully Differentiable Neural Easy-First Taggers"
    Args:
        key: A tensorflow tensor with dimensionality [None, None, key_size]
        context: A tensorflow tensor with dimensionality [None, None, max_num_tokens, token_size]
        hidden_size: Number of units in hidden representation
        depth: Number of csoftmax usages
        projected_align: Using bidirectional lstm for hidden representation of context.
        If true, beetween input and attention mechanism insert layer of bidirectional lstm with dimensionality [hidden_size].
        If false, bidirectional lstm is not used.
    Returns:
        output: Tensor at the output with dimensionality [None, None, depth * hidden_size]


assert axis in (<NUM_LIT:1>, <NUM_LIT:2>)<EOL>n_time_steps = K.int_shape(units)[<NUM_LIT:1>]<EOL>repetitions = [<NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>, <NUM_LIT:1>]<EOL>repetitions[axis] = n_time_steps<EOL>if axis == <NUM_LIT:1>:<EOL><INDENT>expanded = Reshape(target_shape=( (<NUM_LIT:1>,) + K.int_shape(units)[<NUM_LIT:1>:] ))(units)<EOL><DEDENT>else:<EOL><INDENT>expanded = Reshape(target_shape=(K.int_shape(units)[<NUM_LIT:1>:<NUM_LIT:2>] + (<NUM_LIT:1>,) + K.int_shape(units)[<NUM_LIT:2>:]))(units)<EOL><DEDENT>return K.tile(expanded, repetitions)<EOL>

Expand and tile tensor along given axis

Args:
    units: tf tensor with dimensions [batch_size, time_steps, n_input_features]
    axis: axis along which expand and tile. Must be 1 or 2


def additive_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = K.int_shape(units)[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>exp1 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:1>))(units)<EOL>exp2 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:2>))(units)<EOL>units_pairs = Concatenate(axis=<NUM_LIT:3>)([exp1, exp2])<EOL>query = Dense(n_hidden, activation="<STR_LIT>")(units_pairs)<EOL>attention = Dense(<NUM_LIT:1>, activation=lambda x: softmax(x, axis=<NUM_LIT:2>))(query)<EOL>attended_units = Lambda(lambda x: K.sum(attention * x, axis=<NUM_LIT:2>))(exp1)<EOL>output = Dense(n_output_features, activation=activation)(attended_units)<EOL>return output<EOL>

Compute additive self attention for time series of vectors (with batch dimension)
        the formula: score(h_i, h_j) = <v, tanh(W_1 h_i + W_2 h_j)>
        v is a learnable vector of n_hidden dimensionality,
        W_1 and W_2 are learnable [n_hidden, n_input_features] matrices

Args:
    units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
    n_hidden: number of2784131 units in hidden representation of similarity measure
    n_output_features: number of features in output dense layer
    activation: activation at the output

Returns:
    output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


def multiplicative_self_attention(units, n_hidden=None, n_output_features=None, activation=None):

n_input_features = K.int_shape(units)[<NUM_LIT:2>]<EOL>if n_hidden is None:<EOL><INDENT>n_hidden = n_input_features<EOL><DEDENT>if n_output_features is None:<EOL><INDENT>n_output_features = n_input_features<EOL><DEDENT>exp1 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:1>))(units)<EOL>exp2 = Lambda(lambda x: expand_tile(x, axis=<NUM_LIT:2>))(units)<EOL>queries = Dense(n_hidden)(exp1)<EOL>keys = Dense(n_hidden)(exp2)<EOL>scores = Lambda(lambda x: K.sum(queries * x, axis=<NUM_LIT:3>, keepdims=True))(keys)<EOL>attention = Lambda(lambda x: softmax(x, axis=<NUM_LIT:2>))(scores)<EOL>mult = Multiply()([attention, exp1])<EOL>attended_units = Lambda(lambda x: K.sum(x, axis=<NUM_LIT:2>))(mult)<EOL>output = Dense(n_output_features, activation=activation)(attended_units)<EOL>return output<EOL>

Compute multiplicative self attention for time series of vectors (with batch dimension)
the formula: score(h_i, h_j) = <W_1 h_i,  W_2 h_j>,  W_1 and W_2 are learnable matrices
with dimensionality [n_hidden, n_input_features]

Args:
    units: tf tensor with dimensionality [batch_size, time_steps, n_input_features]
    n_hidden: number of units in hidden representation of similarity measure
    n_output_features: number of features in output dense layer
    activation: activation at the output

Returns:
    output: self attended tensor with dimensionality [batch_size, time_steps, n_output_features]


dataset_config = config.get('<STR_LIT>', None)<EOL>if dataset_config:<EOL><INDENT>config.pop('<STR_LIT>')<EOL>ds_type = dataset_config['<STR_LIT:type>']<EOL>if ds_type == '<STR_LIT>':<EOL><INDENT>reader = {'<STR_LIT>': '<STR_LIT>'}<EOL>iterator = {'<STR_LIT>': '<STR_LIT>'}<EOL>config['<STR_LIT>'] = {**dataset_config, **reader}<EOL>config['<STR_LIT>'] = {**dataset_config, **iterator}<EOL><DEDENT>else:<EOL><INDENT>raise Exception("<STR_LIT>".format(ds_type))<EOL><DEDENT><DEDENT>try:<EOL><INDENT>reader_config = dict(config['<STR_LIT>'])<EOL><DEDENT>except KeyError:<EOL><INDENT>raise ConfigError("<STR_LIT>")<EOL><DEDENT>reader = get_model(reader_config.pop('<STR_LIT>'))()<EOL>data_path = reader_config.pop('<STR_LIT>', '<STR_LIT>')<EOL>if isinstance(data_path, list):<EOL><INDENT>data_path = [expand_path(x) for x in data_path]<EOL><DEDENT>else:<EOL><INDENT>data_path = expand_path(data_path)<EOL><DEDENT>return reader.read(data_path, **reader_config)<EOL>

Read data by dataset_reader from specified config.


def get_iterator_from_config(config: dict, data: dict):

iterator_config = config['<STR_LIT>']<EOL>iterator: Union[DataLearningIterator, DataFittingIterator] = from_params(iterator_config,<EOL>data=data)<EOL>return iterator<EOL>

Create iterator (from config) for specified data.


def train_evaluate_model_from_config(config: Union[str, Path, dict],<EOL>iterator: Union[DataLearningIterator, DataFittingIterator] = None, *,<EOL>to_train: bool = True,<EOL>evaluation_targets: Optional[Iterable[str]] = None,<EOL>to_validate: Optional[bool] = None,<EOL>download: bool = False,<EOL>start_epoch_num: Optional[int] = None,<EOL>recursive: bool = False) -> Dict[str, Dict[str, float]]:

config = parse_config(config)<EOL>if download:<EOL><INDENT>deep_download(config)<EOL><DEDENT>if to_train and recursive:<EOL><INDENT>for subconfig in get_all_elems_from_json(config['<STR_LIT>'], '<STR_LIT>'):<EOL><INDENT>log.info(f'<STR_LIT>')<EOL>train_evaluate_model_from_config(subconfig, download=False, recursive=True)<EOL><DEDENT><DEDENT>import_packages(config.get('<STR_LIT>', {}).get('<STR_LIT>', []))<EOL>if iterator is None:<EOL><INDENT>try:<EOL><INDENT>data = read_data_by_config(config)<EOL><DEDENT>except ConfigError as e:<EOL><INDENT>to_train = False<EOL>log.warning(f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>iterator = get_iterator_from_config(config, data)<EOL><DEDENT><DEDENT>if '<STR_LIT:train>' not in config:<EOL><INDENT>log.warning('<STR_LIT>')<EOL><DEDENT>train_config = config.get('<STR_LIT:train>')<EOL>if start_epoch_num is not None:<EOL><INDENT>train_config['<STR_LIT>'] = start_epoch_num<EOL><DEDENT>if '<STR_LIT>' not in train_config and ('<STR_LIT>' in train_config<EOL>or '<STR_LIT>' in train_config):<EOL><INDENT>log.warning('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>train_config['<STR_LIT>'] = []<EOL>if train_config.pop('<STR_LIT>', True):<EOL><INDENT>train_config['<STR_LIT>'].append('<STR_LIT>')<EOL><DEDENT>if train_config.pop('<STR_LIT>', True):<EOL><INDENT>train_config['<STR_LIT>'].append('<STR_LIT:test>')<EOL><DEDENT><DEDENT>trainer_class = get_model(train_config.pop('<STR_LIT>', '<STR_LIT>'))<EOL>trainer = trainer_class(config['<STR_LIT>'], **train_config)<EOL>if to_train:<EOL><INDENT>trainer.train(iterator)<EOL><DEDENT>res = {}<EOL>if iterator is not None:<EOL><INDENT>if to_validate is not None:<EOL><INDENT>if evaluation_targets is None:<EOL><INDENT>log.warning('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>evaluation_targets = ['<STR_LIT:test>']<EOL>if to_validate:<EOL><INDENT>evaluation_targets.append('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.warn('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT><DEDENT>res = trainer.evaluate(iterator, evaluation_targets, print_reports=True)<EOL>trainer.get_chainer().destroy()<EOL><DEDENT>res = {k: v['<STR_LIT>'] for k, v in res.items()}<EOL>return res<EOL>

Make training and evaluation of the model described in corresponding configuration file.


def build_model(config: Union[str, Path, dict], mode: str = '<STR_LIT>',<EOL>load_trained: bool = False, download: bool = False,<EOL>serialized: Optional[bytes] = None) -> Chainer:

config = parse_config(config)<EOL>if serialized:<EOL><INDENT>serialized: list = pickle.loads(serialized)<EOL><DEDENT>if download:<EOL><INDENT>deep_download(config)<EOL><DEDENT>import_packages(config.get('<STR_LIT>', {}).get('<STR_LIT>', []))<EOL>model_config = config['<STR_LIT>']<EOL>model = Chainer(model_config['<STR_LIT>'], model_config['<STR_LIT>'], model_config.get('<STR_LIT>'))<EOL>for component_config in model_config['<STR_LIT>']:<EOL><INDENT>if load_trained and ('<STR_LIT>' in component_config or '<STR_LIT>' in component_config):<EOL><INDENT>try:<EOL><INDENT>component_config['<STR_LIT>'] = component_config['<STR_LIT>']<EOL><DEDENT>except KeyError:<EOL><INDENT>log.warning('<STR_LIT>'<EOL>.format(component_config.get('<STR_LIT>', component_config.get('<STR_LIT>', '<STR_LIT>'))))<EOL><DEDENT><DEDENT>if serialized and '<STR_LIT>' in component_config:<EOL><INDENT>component_serialized = serialized.pop(<NUM_LIT:0>)<EOL><DEDENT>else:<EOL><INDENT>component_serialized = None<EOL><DEDENT>component = from_params(component_config, mode=mode, serialized=component_serialized)<EOL>if '<STR_LIT>' in component_config:<EOL><INDENT>c_in = component_config['<STR_LIT>']<EOL>c_out = component_config['<STR_LIT>']<EOL>in_y = component_config.get('<STR_LIT>', None)<EOL>main = component_config.get('<STR_LIT>', False)<EOL>model.append(component, c_in, c_out, in_y, main)<EOL><DEDENT><DEDENT>return model<EOL>

Build and return the model described in corresponding configuration file.


def interact_model(config: Union[str, Path, dict]) -> None:

model = build_model(config)<EOL>while True:<EOL><INDENT>args = []<EOL>for in_x in model.in_x:<EOL><INDENT>args.append((input('<STR_LIT>'.format(in_x)),))<EOL>if args[-<NUM_LIT:1>][<NUM_LIT:0>] in {'<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT:q>'}:<EOL><INDENT>return<EOL><DEDENT><DEDENT>pred = model(*args)<EOL>if len(model.out_params) > <NUM_LIT:1>:<EOL><INDENT>pred = zip(*pred)<EOL><DEDENT>print('<STR_LIT>', *pred)<EOL><DEDENT>

Start interaction with the model described in corresponding configuration file.


def predict_on_stream(config: Union[str, Path, dict], batch_size: int = <NUM_LIT:1>, file_path: Optional[str] = None) -> None:

if file_path is None or file_path == '<STR_LIT:->':<EOL><INDENT>if sys.stdin.isatty():<EOL><INDENT>raise RuntimeError('<STR_LIT>')<EOL><DEDENT>f = sys.stdin<EOL><DEDENT>else:<EOL><INDENT>f = open(file_path, encoding='<STR_LIT:utf8>')<EOL><DEDENT>model: Chainer = build_model(config)<EOL>args_count = len(model.in_x)<EOL>while True:<EOL><INDENT>batch = list((l.strip() for l in islice(f, batch_size * args_count)))<EOL>if not batch:<EOL><INDENT>break<EOL><DEDENT>args = []<EOL>for i in range(args_count):<EOL><INDENT>args.append(batch[i::args_count])<EOL><DEDENT>res = model(*args)<EOL>if len(model.out_params) == <NUM_LIT:1>:<EOL><INDENT>res = [res]<EOL><DEDENT>for res in zip(*res):<EOL><INDENT>res = json.dumps(res, ensure_ascii=False)<EOL>print(res, flush=True)<EOL><DEDENT><DEDENT>if f is not sys.stdin:<EOL><INDENT>f.close()<EOL><DEDENT>

Make a prediction with the component described in corresponding configuration file.


def _parse_config_property(item: _T, variables: Dict[str, Union[str, Path, float, bool, None]]) -> _T:

if isinstance(item, str):<EOL><INDENT>return item.format(**variables)<EOL><DEDENT>elif isinstance(item, list):<EOL><INDENT>return [_parse_config_property(item, variables) for item in item]<EOL><DEDENT>elif isinstance(item, dict):<EOL><INDENT>return {k: _parse_config_property(v, variables) for k, v in item.items()}<EOL><DEDENT>else:<EOL><INDENT>return item<EOL><DEDENT>

Recursively apply config's variables values to its property


def parse_config(config: Union[str, Path, dict]) -> dict:

if isinstance(config, (str, Path)):<EOL><INDENT>config = read_json(find_config(config))<EOL><DEDENT>variables = {<EOL>'<STR_LIT>': os.getenv(f'<STR_LIT>', Path(__file__).parent.parent.parent)<EOL>}<EOL>for name, value in config.get('<STR_LIT>', {}).get('<STR_LIT>', {}).items():<EOL><INDENT>env_name = f'<STR_LIT>'<EOL>if env_name in os.environ:<EOL><INDENT>value = os.getenv(env_name)<EOL><DEDENT>variables[name] = value.format(**variables)<EOL><DEDENT>return _parse_config_property(config, variables)<EOL>

Read config's variables and apply their values to all its properties


def expand_path(path: Union[str, Path]) -> Path:

return Path(path).expanduser().resolve()<EOL>

Convert relative paths to absolute with resolving user directory.


def import_packages(packages: list) -> None:

for package in packages:<EOL><INDENT>__import__(package)<EOL><DEDENT>

Import packages from list to execute their code.


def from_params(params: Dict, mode: str = '<STR_LIT>', serialized: Any = None, **kwargs) -> Component:

<EOL>config_params = {k: _resolve(v) for k, v in params.items()}<EOL>if '<STR_LIT>' in config_params:<EOL><INDENT>try:<EOL><INDENT>component = _refs[config_params['<STR_LIT>']]<EOL>if serialized is not None:<EOL><INDENT>component.deserialize(serialized)<EOL><DEDENT>return component<EOL><DEDENT>except KeyError:<EOL><INDENT>e = ConfigError('<STR_LIT>'<EOL>.format(id=config_params['<STR_LIT>']))<EOL>log.exception(e)<EOL>raise e<EOL><DEDENT><DEDENT>elif '<STR_LIT>' in config_params:<EOL><INDENT>from deeppavlov.core.commands.infer import build_model<EOL>refs = _refs.copy()<EOL>_refs.clear()<EOL>config = parse_config(expand_path(config_params['<STR_LIT>']))<EOL>model = build_model(config, serialized=serialized)<EOL>_refs.clear()<EOL>_refs.update(refs)<EOL>try:<EOL><INDENT>_refs[config_params['<STR_LIT:id>']] = model <EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT>return model<EOL><DEDENT>cls_name = config_params.pop('<STR_LIT>', None)<EOL>if not cls_name:<EOL><INDENT>e = ConfigError('<STR_LIT>')<EOL>log.exception(e)<EOL>raise e<EOL><DEDENT>cls = get_model(cls_name)<EOL>config_params = {k: _init_param(v, mode) for k, v in config_params.items()}<EOL>try:<EOL><INDENT>spec = inspect.getfullargspec(cls)<EOL>if '<STR_LIT>' in spec.args+spec.kwonlyargs or spec.varkw is not None:<EOL><INDENT>kwargs['<STR_LIT>'] = mode<EOL><DEDENT>component = cls(**dict(config_params, **kwargs))<EOL>try:<EOL><INDENT>_refs[config_params['<STR_LIT:id>']] = component<EOL><DEDENT>except KeyError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>log.exception("<STR_LIT>".format(cls))<EOL>raise<EOL><DEDENT>if serialized is not None:<EOL><INDENT>component.deserialize(serialized)<EOL><DEDENT>return component<EOL>

Builds and returns the Component from corresponding dictionary of parameters.


global _gpu_available<EOL>if _gpu_available is None:<EOL><INDENT>sess_config = tf.ConfigProto()<EOL>sess_config.gpu_options.allow_growth = True<EOL>try:<EOL><INDENT>with tf.Session(config=sess_config):<EOL><INDENT>device_list = device_lib.list_local_devices()<EOL>_gpu_available = any(device.device_type == '<STR_LIT>' for device in device_list)<EOL><DEDENT><DEDENT>except AttributeError as e:<EOL><INDENT>log.warning(f'<STR_LIT>')<EOL>_gpu_available = False<EOL><DEDENT><DEDENT>return _gpu_available<EOL>

r"""Return True if at least one GPU is available


def __init__(self,<EOL>prefix="<STR_LIT>",<EOL>seed: int = None,<EOL>**kwargs):

self.basic_config = deepcopy(kwargs)<EOL>self.prefix = prefix<EOL>self.paths_to_params = []<EOL>for search_type in [prefix + "<STR_LIT>", prefix + "<STR_LIT>", prefix + "<STR_LIT>"]:<EOL><INDENT>for path_ in self.find_model_path(self.basic_config, search_type):<EOL><INDENT>self.paths_to_params.append(path_)<EOL><DEDENT><DEDENT>self.n_params = len(self.paths_to_params)<EOL>self.eps = <NUM_LIT><EOL>if seed is None:<EOL><INDENT>pass<EOL><DEDENT>else:<EOL><INDENT>np.random.seed(seed)<EOL>random.seed(seed)<EOL><DEDENT>

Initialize evolution with random population


def find_model_path(self, config: dict, key_model: str, path: list = []) -> Generator:

config_pointer = config<EOL>if isinstance(config_pointer, dict) and key_model in config_pointer.keys():<EOL><INDENT>yield path<EOL><DEDENT>else:<EOL><INDENT>if isinstance(config_pointer, dict):<EOL><INDENT>for key in list(config_pointer.keys()):<EOL><INDENT>for path_ in self.find_model_path(config_pointer[key], key_model, path + [key]):<EOL><INDENT>yield path_<EOL><DEDENT><DEDENT><DEDENT>elif isinstance(config_pointer, list):<EOL><INDENT>for i in range(len(config_pointer)):<EOL><INDENT>for path_ in self.find_model_path(config_pointer[i], key_model, path + [i]):<EOL><INDENT>yield path_<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Find paths to all dictionaries in config that contain key 'key_model'

Args:
    config: dictionary
    key_model: key of sub-dictionary to be found
    path: list of keys and/or integers (for list) with relative path (needed for recursion)

Returns:
    path in config -- list of keys (strings and integers)


@staticmethod<EOL><INDENT>def insert_value_or_dict_into_config(config: dict, path: list,<EOL>value: [int, float, str, bool, list, dict, np.ndarray]) -> None:<DEDENT>

config_pointer = config<EOL>for el in path[:-<NUM_LIT:1>]:<EOL><INDENT>if isinstance(config_pointer, dict):<EOL><INDENT>config_pointer = config_pointer.setdefault(el, {})<EOL><DEDENT>elif isinstance(config_pointer, list):<EOL><INDENT>config_pointer = config_pointer[el]<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>config_pointer[path[-<NUM_LIT:1>]] = value<EOL>

Insert value to dictionary determined by path[:-1] in field with key path[-1]

Args:
    config: dictionary
    path: list of keys and/or integers (for list)
    value: value to be inserted

Returns:
    config with inserted value


@staticmethod<EOL><INDENT>def get_value_from_config(config: dict, path: list) -> Any:<DEDENT>

config_copy = deepcopy(config)<EOL>config_pointer = config_copy<EOL>for el in path[:-<NUM_LIT:1>]:<EOL><INDENT>if isinstance(config_pointer, dict):<EOL><INDENT>config_pointer = config_pointer.setdefault(el, {})<EOL><DEDENT>elif isinstance(config_pointer, list):<EOL><INDENT>config_pointer = config_pointer[el]<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>return config_pointer[path[-<NUM_LIT:1>]]<EOL>

Return value of config element determined by path

Args:
    config: dictionary
    path: list of keys and/or integers (for list)

Returns:
    value


@staticmethod<EOL><INDENT>def remove_key_from_config(config: dict, path: list) -> Tuple[dict, Any]:<DEDENT>

config_copy = deepcopy(config)<EOL>config_pointer = config_copy<EOL>for el in path[:-<NUM_LIT:1>]:<EOL><INDENT>if isinstance(config_pointer, dict):<EOL><INDENT>config_pointer = config_pointer.setdefault(el, {})<EOL><DEDENT>elif isinstance(config_pointer, list):<EOL><INDENT>config_pointer = config_pointer[el]<EOL><DEDENT>else:<EOL><INDENT>pass<EOL><DEDENT><DEDENT>value = config_pointer.pop(path[-<NUM_LIT:1>])<EOL>return config_copy, value<EOL>

Remove config element determined by path

Args:
    config: dictionary
    path: list of keys and/or integers (for list)

Returns:
    dictionary without value from path, value from path


def initialize_params_in_config(self, basic_config: dict, paths: List[list]) -> dict:

config = deepcopy(basic_config)<EOL>for path_ in paths:<EOL><INDENT>param_name = path_[-<NUM_LIT:1>]<EOL>value = self.get_value_from_config(basic_config, path_)<EOL>if isinstance(value, dict):<EOL><INDENT>if (value.get(self.prefix + "<STR_LIT>") or<EOL>value.get(self.prefix + "<STR_LIT>") or<EOL>value.get(self.prefix + "<STR_LIT>")):<EOL><INDENT>self.insert_value_or_dict_into_config(<EOL>config, path_,<EOL>self.sample_params(**{param_name: deepcopy(value)})[param_name])<EOL><DEDENT><DEDENT><DEDENT>return config<EOL>

Randomly initialize all the changable parameters in config

Args:
    basic_config: config where changable parameters are dictionaries with keys
        ``prefix`_range`, ``prefix`_bool`, ``prefix`_choice`
    paths: list of paths to changable parameters

Returns:
    config


def sample_params(self, **params) -> dict:

if not params:<EOL><INDENT>return {}<EOL><DEDENT>else:<EOL><INDENT>params_copy = deepcopy(params)<EOL><DEDENT>params_sample = dict()<EOL>for param, param_val in params_copy.items():<EOL><INDENT>if isinstance(param_val, dict):<EOL><INDENT>if self.prefix + '<STR_LIT>' in param_val and param_val[self.prefix + '<STR_LIT>']:<EOL><INDENT>sample = bool(random.choice([True, False]))<EOL><DEDENT>elif self.prefix + '<STR_LIT>' in param_val:<EOL><INDENT>sample = self._sample_from_ranges(param_val)<EOL><DEDENT>elif self.prefix + '<STR_LIT>' in param_val:<EOL><INDENT>sample = random.choice(param_val[self.prefix + '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>sample = param_val<EOL><DEDENT>params_sample[param] = sample<EOL><DEDENT>else:<EOL><INDENT>params_sample[param] = params_copy[param]<EOL><DEDENT><DEDENT>return params_sample<EOL>

Sample parameters according to the given possible values

Args:
    **params: dictionary like {"param_0": {"`prefix`_range": [0, 10]},
                               "param_1": {"`prefix`_range": [0, 10], "discrete": true},
                               "param_2": {"`prefix`_range": [0, 1], "scale": "log"},
                               "param_3": {"`prefix`_bool": true},
                               "param_4": {"`prefix`_choice": [0, 1, 2, 3]}}

Returns:
    dictionary with randomly sampled parameters


def _sample_from_ranges(self, opts: dict) -> [int, float]:

from_ = opts[self.prefix + '<STR_LIT>'][<NUM_LIT:0>]<EOL>to_ = opts[self.prefix + '<STR_LIT>'][<NUM_LIT:1>]<EOL>if opts.get('<STR_LIT>', None) == '<STR_LIT>':<EOL><INDENT>sample = self._sample_log(from_, to_)<EOL><DEDENT>else:<EOL><INDENT>sample = np.random.uniform(from_, to_)<EOL><DEDENT>if opts.get('<STR_LIT>', False):<EOL><INDENT>sample = int(np.round(sample))<EOL><DEDENT>return sample<EOL>

Sample parameters from ranges

Args:
    opts: dictionary  {"`prefix`_range": [0, 10]} or \
                      {"`prefix`_range": [0, 10], "discrete": true} or \
                      {"`prefix`_range": [0, 1], "scale": "log"}

Returns:
    random parameter value from range


@staticmethod<EOL><INDENT>def _sample_log(from_: float = <NUM_LIT:0.>, to_: float = <NUM_LIT:1.>) -> float:<DEDENT>

sample = np.exp(np.random.uniform(np.log(from_), np.log(to_)))<EOL>return float(sample)<EOL>

Sample parameters from ranges with log scale

Args:
    from_: lower boundary of values
    to_:  upper boundary of values

Returns:
    random parameters value from range with log scale


def fn_from_str(name: str) -> Callable[..., Any]:

try:<EOL><INDENT>module_name, fn_name = name.split('<STR_LIT::>')<EOL><DEDENT>except ValueError:<EOL><INDENT>raise ConfigError('<STR_LIT>'<EOL>.format(name))<EOL><DEDENT>return getattr(importlib.import_module(module_name), fn_name)<EOL>

Returns a function object with the name given in string.


def register_metric(metric_name: str) -> Callable[..., Any]:

def decorate(fn):<EOL><INDENT>fn_name = fn.__module__ + '<STR_LIT::>' + fn.__name__<EOL>if metric_name in _REGISTRY and _REGISTRY[metric_name] != fn_name:<EOL><INDENT>log.warning('<STR_LIT>'<EOL>.format(metric_name))<EOL><DEDENT>_REGISTRY[metric_name] = fn_name<EOL>return fn<EOL><DEDENT>return decorate<EOL>


def get_metric_by_name(name: str) -> Callable[..., Any]:

if name not in _REGISTRY:<EOL><INDENT>raise ConfigError(f'<STR_LIT>')<EOL><DEDENT>return fn_from_str(_REGISTRY[name])<EOL>

Returns a metric callable with a corresponding name.


populate_settings_dir()<EOL>return _settings_path<EOL>

Return an absolute path to the DeepPavlov settings directory


def populate_settings_dir(force: bool = False) -> bool:

res = False<EOL>if _default_settings_path == _settings_path:<EOL><INDENT>return res<EOL><DEDENT>for src in list(_default_settings_path.glob('<STR_LIT>')):<EOL><INDENT>dest = _settings_path / src.relative_to(_default_settings_path)<EOL>if not force and dest.exists():<EOL><INDENT>continue<EOL><DEDENT>res = True<EOL>dest.parent.mkdir(parents=True, exist_ok=True)<EOL>shutil.copy(src, dest)<EOL><DEDENT>return res<EOL>

Populate settings directory with default settings files

Args:
    force: if ``True``, replace existing settings files with default ones

Returns:
    ``True`` if any files were copied and ``False`` otherwise


try:<EOL><INDENT>module_name, cls_name = name.split('<STR_LIT::>')<EOL><DEDENT>except ValueError:<EOL><INDENT>raise ConfigError('<STR_LIT>'<EOL>.format(name))<EOL><DEDENT>return getattr(importlib.import_module(module_name), cls_name)<EOL>

Returns a class object with the name given as a string.


def decorate(model_cls: type, reg_name: str = None) -> type:<EOL><INDENT>model_name = reg_name or short_name(model_cls)<EOL>global _REGISTRY<EOL>cls_name = model_cls.__module__ + '<STR_LIT::>' + model_cls.__name__<EOL>if model_name in _REGISTRY and _REGISTRY[model_name] != cls_name:<EOL><INDENT>logger.warning('<STR_LIT>'.format(model_name))<EOL><DEDENT>_REGISTRY[model_name] = cls_name<EOL>return model_cls<EOL><DEDENT>return lambda model_cls_name: decorate(model_cls_name, name)<EOL>

Register classes that could be initialized from JSON configuration file.
If name is not passed, the class name is converted to snake-case.


return cls.__name__.split('<STR_LIT:.>')[-<NUM_LIT:1>]<EOL>

Returns just a class name (without package and module specification).


if name not in _REGISTRY:<EOL><INDENT>if '<STR_LIT::>' not in name:<EOL><INDENT>raise ConfigError("<STR_LIT>".format(name))<EOL><DEDENT>return cls_from_str(name)<EOL><DEDENT>return cls_from_str(_REGISTRY[name])<EOL>

Returns a registered class object with the name given in the string.


return list(_REGISTRY)<EOL>

Returns a list of names of registered classes.


def read(self, data_path: str, *args, **kwargs) -> Dict[str, List[Tuple[Any, Any]]]:

raise NotImplementedError<EOL>

Reads a file from a path and returns data as a list of tuples of inputs and correct outputs
         for every data type in ``train``, ``valid`` and ``test``.


def download(dest_file_path: [List[Union[str, Path]]], source_url: str, force_download=True):

if isinstance(dest_file_path, list):<EOL><INDENT>dest_file_paths = [Path(path) for path in dest_file_path]<EOL><DEDENT>else:<EOL><INDENT>dest_file_paths = [Path(dest_file_path).absolute()]<EOL><DEDENT>if not force_download:<EOL><INDENT>to_check = list(dest_file_paths)<EOL>dest_file_paths = []<EOL>for p in to_check:<EOL><INDENT>if p.exists():<EOL><INDENT>log.info(f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>dest_file_paths.append(p)<EOL><DEDENT><DEDENT><DEDENT>if dest_file_paths:<EOL><INDENT>cache_dir = os.getenv('<STR_LIT>')<EOL>cached_exists = False<EOL>if cache_dir:<EOL><INDENT>first_dest_path = Path(cache_dir) / md5(source_url.encode('<STR_LIT:utf8>')).hexdigest()[:<NUM_LIT:15>]<EOL>cached_exists = first_dest_path.exists()<EOL><DEDENT>else:<EOL><INDENT>first_dest_path = dest_file_paths.pop()<EOL><DEDENT>if not cached_exists:<EOL><INDENT>first_dest_path.parent.mkdir(parents=True, exist_ok=True)<EOL>simple_download(source_url, first_dest_path)<EOL><DEDENT>else:<EOL><INDENT>log.info(f'<STR_LIT>')<EOL><DEDENT>for dest_path in dest_file_paths:<EOL><INDENT>dest_path.parent.mkdir(parents=True, exist_ok=True)<EOL>shutil.copy(str(first_dest_path), str(dest_path))<EOL><DEDENT><DEDENT>

Download a file from URL to one or several target locations

    Args:
        dest_file_path: path or list of paths to the file destination files (including file name)
        source_url: the source URL
        force_download: download file if it already exists, or not


file_path = Path(file_path)<EOL>if extract_folder is None:<EOL><INDENT>extract_folder = file_path.parent<EOL><DEDENT>extract_folder = Path(extract_folder)<EOL>tar = tarfile.open(file_path)<EOL>tar.extractall(extract_folder)<EOL>tar.close()<EOL>

Simple tar archive extractor

    Args:
        file_path: path to the tar file to be extracted
        extract_folder: folder to which the files will be extracted


def ungzip(file_path, extract_path: Path = None):

CHUNK = <NUM_LIT:16> * <NUM_LIT><EOL>file_path = Path(file_path)<EOL>extract_path = extract_path or file_path.with_suffix('<STR_LIT>')<EOL>with gzip.open(file_path, '<STR_LIT:rb>') as fin, extract_path.open('<STR_LIT:wb>') as fout:<EOL><INDENT>while True:<EOL><INDENT>block = fin.read(CHUNK)<EOL>if not block:<EOL><INDENT>break<EOL><DEDENT>fout.write(block)<EOL><DEDENT><DEDENT>

Simple .gz archive extractor

        Args:
            file_path: path to the gzip file to be extracted
            extract_path: path where the file will be extracted


def download_decompress(url: str, download_path: [Path, str], extract_paths=None):

file_name = Path(urlparse(url).path).name<EOL>download_path = Path(download_path)<EOL>if extract_paths is None:<EOL><INDENT>extract_paths = [download_path]<EOL><DEDENT>elif isinstance(extract_paths, list):<EOL><INDENT>extract_paths = [Path(path) for path in extract_paths]<EOL><DEDENT>else:<EOL><INDENT>extract_paths = [Path(extract_paths)]<EOL><DEDENT>cache_dir = os.getenv('<STR_LIT>')<EOL>extracted = False<EOL>if cache_dir:<EOL><INDENT>cache_dir = Path(cache_dir)<EOL>url_hash = md5(url.encode('<STR_LIT:utf8>')).hexdigest()[:<NUM_LIT:15>]<EOL>arch_file_path = cache_dir / url_hash<EOL>extracted_path = cache_dir / (url_hash + '<STR_LIT>')<EOL>extracted = extracted_path.exists()<EOL>if not extracted and not arch_file_path.exists():<EOL><INDENT>simple_download(url, arch_file_path)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>arch_file_path = download_path / file_name<EOL>simple_download(url, arch_file_path)<EOL>extracted_path = extract_paths.pop()<EOL><DEDENT>if not extracted:<EOL><INDENT>log.info('<STR_LIT>'.format(arch_file_path, extracted_path))<EOL>extracted_path.mkdir(parents=True, exist_ok=True)<EOL>if file_name.endswith('<STR_LIT>'):<EOL><INDENT>untar(arch_file_path, extracted_path)<EOL><DEDENT>elif file_name.endswith('<STR_LIT>'):<EOL><INDENT>ungzip(arch_file_path, extracted_path / Path(file_name).with_suffix('<STR_LIT>').name)<EOL><DEDENT>elif file_name.endswith('<STR_LIT>'):<EOL><INDENT>with zipfile.ZipFile(arch_file_path, '<STR_LIT:r>') as zip_ref:<EOL><INDENT>zip_ref.extractall(extracted_path)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>raise RuntimeError(f'<STR_LIT>')<EOL><DEDENT>if not cache_dir:<EOL><INDENT>arch_file_path.unlink()<EOL><DEDENT><DEDENT>for extract_path in extract_paths:<EOL><INDENT>for src in extracted_path.iterdir():<EOL><INDENT>dest = extract_path / src.name<EOL>if src.is_dir():<EOL><INDENT>copytree(src, dest)<EOL><DEDENT>else:<EOL><INDENT>extract_path.mkdir(parents=True, exist_ok=True)<EOL>shutil.copy(str(src), str(dest))<EOL><DEDENT><DEDENT><DEDENT>

Download and extract .tar.gz or .gz file to one or several target locations.
    The archive is deleted if extraction was successful.

    Args:
        url: URL for file downloading
        download_path: path to the directory where downloaded file will be stored
        until the end of extraction
        extract_paths: path or list of paths where contents of archive will be extracted


def update_dict_recursive(editable_dict: dict, editing_dict: dict) -> None:

for k, v in editing_dict.items():<EOL><INDENT>if isinstance(v, collections.Mapping):<EOL><INDENT>update_dict_recursive(editable_dict.get(k, {}), v)<EOL><DEDENT>else:<EOL><INDENT>editable_dict[k] = v<EOL><DEDENT><DEDENT>

Updates dict recursively

    You need to use this function to update dictionary if depth of editing_dict is more then 1

    Args:
        editable_dict: dictionary, that will be edited
        editing_dict: dictionary, that contains edits
    Returns:
        None


scheme, netloc, path, query_string, fragment = urlsplit(url)<EOL>path += '<STR_LIT>'<EOL>return urlunsplit((scheme, netloc, path, query_string, fragment))<EOL>

Given a file URL, return a md5 query of the file

    Args:
        url: a given URL
    Returns:
        URL of the md5 file


def set_query_parameter(url, param_name, param_value):

scheme, netloc, path, query_string, fragment = urlsplit(url)<EOL>query_params = parse_qs(query_string)<EOL>query_params[param_name] = [param_value]<EOL>new_query_string = urlencode(query_params, doseq=True)<EOL>return urlunsplit((scheme, netloc, path, new_query_string, fragment))<EOL>

Given a URL, set or replace a query parameter and return the modified URL.

    Args:
        url: a given  URL
        param_name: the parameter name to add
        param_value: the parameter value
    Returns:
        URL with the added parameter


def gen_batches(self, batch_size: int, data_type: str = '<STR_LIT:train>',<EOL>shuffle: bool = None) -> Iterator[Tuple[tuple, tuple]]:

if shuffle is None:<EOL><INDENT>shuffle = self.shuffle<EOL><DEDENT>data = self.data[data_type]<EOL>data_len = len(data)<EOL>if data_len == <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>order = list(range(data_len))<EOL>if shuffle:<EOL><INDENT>self.random.shuffle(order)<EOL><DEDENT>if batch_size < <NUM_LIT:0>:<EOL><INDENT>batch_size = data_len<EOL><DEDENT>for i in range((data_len - <NUM_LIT:1>) // batch_size + <NUM_LIT:1>):<EOL><INDENT>yield tuple(zip(*[data[o] for o in order[i * batch_size:(i + <NUM_LIT:1>) * batch_size]]))<EOL><DEDENT>

Generate batches of inputs and expected output to train neural networks

        Args:
            batch_size: number of samples in batch
            data_type: can be either 'train', 'test', or 'valid'
            shuffle: whether to shuffle dataset before batching

        Yields:
             a tuple of a batch of inputs and a batch of expected outputs


def get_instances(self, data_type: str = '<STR_LIT:train>') -> Tuple[tuple, tuple]:

data = self.data[data_type]<EOL>return tuple(zip(*data))<EOL>

Get all data for a selected data type

        Args:
            data_type (str): can be either ``'train'``, ``'test'``, ``'valid'`` or ``'all'``

        Returns:
             a tuple of all inputs for a data type and all expected outputs for a data type


return list(range(len(self.data)))<EOL>

Generate doc ids.

        Returns: doc ids


def get_doc_content(self, doc_id: Any) -> Optional[str]:

return self.data[doc_id]<EOL>

Get doc content by id.

        Args:
            doc_id: an id for a doc which content should be extracted

        Returns:
            doc content as a string if id exists or raise an error


def gen_batches(self, batch_size: int, shuffle: bool = None)-> Generator[Tuple[List[str], List[int]], Any, None]:

if shuffle is None:<EOL><INDENT>shuffle = self.shuffle<EOL><DEDENT>if shuffle:<EOL><INDENT>_doc_ids = self.random.sample(self.doc_ids, len(self.doc_ids))<EOL><DEDENT>else:<EOL><INDENT>_doc_ids = self.doc_ids<EOL><DEDENT>if batch_size > <NUM_LIT:0>:<EOL><INDENT>batches = [_doc_ids[i:i + batch_size] for i in<EOL>range(<NUM_LIT:0>, len(_doc_ids), batch_size)]<EOL><DEDENT>else:<EOL><INDENT>batches = [_doc_ids]<EOL><DEDENT>for i, doc_ids in enumerate(batches):<EOL><INDENT>docs = [self.get_doc_content(doc_id) for doc_id in doc_ids]<EOL>yield docs, doc_ids<EOL><DEDENT>

Gen batches of documents.

        Args:
            batch_size: a number of samples in a single batch
            shuffle: whether to shuffle data during batching

        Yields:
            generated tuple of documents and their ids


doc_ids = list(self.doc_ids)<EOL>docs = [self.get_doc_content(doc_id) for doc_id in doc_ids]<EOL>return docs, doc_ids<EOL>


data = request.get_json()<EOL>text = data['<STR_LIT>'].get('<STR_LIT>', '<STR_LIT>').strip()<EOL>payload = data['<STR_LIT>'].get('<STR_LIT>')<EOL>session_id = data['<STR_LIT>']['<STR_LIT>']<EOL>user_id = data['<STR_LIT>']['<STR_LIT>']<EOL>message_id = data['<STR_LIT>']['<STR_LIT>']<EOL>dialog_id = DialogID(user_id, session_id)<EOL>response = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': True,<EOL>'<STR_LIT:text>': '<STR_LIT>'<EOL>},<EOL>"<STR_LIT>": {<EOL>'<STR_LIT>': session_id,<EOL>'<STR_LIT>': message_id,<EOL>'<STR_LIT>': user_id<EOL>},<EOL>'<STR_LIT:version>': '<STR_LIT:1.0>'<EOL>}<EOL>agent_response: Union[str, RichMessage] = agent([payload or text], [dialog_id])[<NUM_LIT:0>]<EOL>if isinstance(agent_response, RichMessage):<EOL><INDENT>response['<STR_LIT>']['<STR_LIT:text>'] = '<STR_LIT:\n>'.join([j['<STR_LIT:content>']<EOL>for j in agent_response.json()<EOL>if j['<STR_LIT:type>'] == '<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>response['<STR_LIT>']['<STR_LIT:text>'] = str(agent_response)<EOL><DEDENT>return jsonify(response), <NUM_LIT:200><EOL>

Exchange messages between basic pipelines and the Yandex.Dialogs service.
If the pipeline returns multiple values, only the first one is forwarded to Yandex.


while True:<EOL><INDENT>request = self.input_queue.get()<EOL>response = self._handle_request(request)<EOL>self.output_queue.put(response)<EOL><DEDENT>


def _del_conversation(self, conversation_key: str) -> None:

if conversation_key in self.conversations.keys():<EOL><INDENT>del self.conversations[conversation_key]<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>

Deletes Conversation instance.

        Args:
            conversation_key: Conversation key.


<EOL>agent = self.agent_generator()<EOL>return agent<EOL>

Initiates Alexa skill agent from agent generator


self.timer = Timer(REFRESH_VALID_CERTS_PERIOD_SECS, self._refresh_valid_certs)<EOL>self.timer.start()<EOL>expired_certificates = []<EOL>for valid_cert_url, valid_cert in self.valid_certificates.items():<EOL><INDENT>valid_cert: ValidatedCert = valid_cert<EOL>cert_expiration_time: datetime = valid_cert.expiration_timestamp<EOL>if datetime.utcnow() > cert_expiration_time:<EOL><INDENT>expired_certificates.append(valid_cert_url)<EOL><DEDENT><DEDENT>for expired_cert_url in expired_certificates:<EOL><INDENT>del self.valid_certificates[expired_cert_url]<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>

Conducts cleanup of periodical certificates with expired validation.


def _verify_request(self, signature_chain_url: str, signature: str, request_body: bytes) -> bool:

if signature_chain_url not in self.valid_certificates.keys():<EOL><INDENT>amazon_cert: X509 = verify_cert(signature_chain_url)<EOL>if amazon_cert:<EOL><INDENT>amazon_cert_lifetime: timedelta = self.config['<STR_LIT>']<EOL>expiration_timestamp = datetime.utcnow() + amazon_cert_lifetime<EOL>validated_cert = ValidatedCert(cert=amazon_cert, expiration_timestamp=expiration_timestamp)<EOL>self.valid_certificates[signature_chain_url] = validated_cert<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>validated_cert: ValidatedCert = self.valid_certificates[signature_chain_url]<EOL>amazon_cert: X509 = validated_cert.cert<EOL><DEDENT>if verify_signature(amazon_cert, signature, request_body):<EOL><INDENT>result = True<EOL><DEDENT>else:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>result = False<EOL><DEDENT>return result<EOL>

Conducts series of Alexa request verifications against Amazon Alexa requirements.

        Args:
            signature_chain_url: Signature certificate URL from SignatureCertChainUrl HTTP header.
            signature: Base64 decoded Alexa request signature from Signature HTTP header.
            request_body: full HTTPS request body
        Returns:
            result: True if verification was successful, False if not.


def _handle_request(self, request: dict) -> dict:

request_body: bytes = request['<STR_LIT>']<EOL>signature_chain_url: str = request['<STR_LIT>']<EOL>signature: str = request['<STR_LIT>']<EOL>alexa_request: dict = request['<STR_LIT>']<EOL>if not self._verify_request(signature_chain_url, signature, request_body):<EOL><INDENT>return {'<STR_LIT:error>': '<STR_LIT>'}<EOL><DEDENT>timestamp_str = alexa_request['<STR_LIT>']['<STR_LIT>']<EOL>timestamp_datetime = datetime.strptime(timestamp_str, '<STR_LIT>')<EOL>now = datetime.utcnow()<EOL>delta = now - timestamp_datetime if now >= timestamp_datetime else timestamp_datetime - now<EOL>if abs(delta.seconds) > REQUEST_TIMESTAMP_TOLERANCE_SECS:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return {'<STR_LIT:error>': '<STR_LIT>'}<EOL><DEDENT>conversation_key = alexa_request['<STR_LIT>']['<STR_LIT:user>']['<STR_LIT>']<EOL>if conversation_key not in self.conversations.keys():<EOL><INDENT>if self.config['<STR_LIT>']:<EOL><INDENT>conv_agent = self._init_agent()<EOL>log.info('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>conv_agent = self.agent<EOL><DEDENT>self.conversations[conversation_key] =Conversation(config=self.config,<EOL>agent=conv_agent,<EOL>conversation_key=conversation_key,<EOL>self_destruct_callback=lambda: self._del_conversation(conversation_key))<EOL>log.info(f'<STR_LIT>')<EOL><DEDENT>conversation = self.conversations[conversation_key]<EOL>response = conversation.handle_request(alexa_request)<EOL>return response<EOL>

Processes Alexa requests from skill server and returns responses to Alexa.

        Args:
            request: Dict with Alexa request payload and metadata.
        Returns:
            result: Alexa formatted or error response.


parsed = urlsplit(url)<EOL>scheme: str = parsed.scheme<EOL>netloc: str = parsed.netloc<EOL>path: str = parsed.path<EOL>try:<EOL><INDENT>port = parsed.port<EOL><DEDENT>except ValueError:<EOL><INDENT>port = None<EOL><DEDENT>result = (scheme.lower() == '<STR_LIT>' and<EOL>netloc.lower().split('<STR_LIT::>')[<NUM_LIT:0>] == '<STR_LIT>' and<EOL>path.startswith('<STR_LIT>') and<EOL>(port == <NUM_LIT> or port is None))<EOL>return result<EOL>

Verify signature certificate URL against Amazon Alexa requirements.

    Each call of Agent passes incoming utterances batch through skills filter,
    agent skills, skills processor. Batch of dialog IDs can be provided, in
    other case utterances indexes in incoming batch are used as dialog IDs.

    Args:
        url: Signature certificate URL from SignatureCertChainUrl HTTP header.

    Returns:
        result: True if verification was successful, False if not.


def extract_certs(certs_txt: str) -> List[crypto.X509]:

pattern = r'<STR_LIT>'<EOL>certs_txt = re.findall(pattern, certs_txt, flags=re.DOTALL)<EOL>certs = [crypto.load_certificate(crypto.FILETYPE_PEM, cert_txt) for cert_txt in certs_txt]<EOL>return certs<EOL>

Extracts pycrypto X509 objects from SSL certificates chain string.

    Args:
        certs_txt: SSL certificates chain string.

    Returns:
        result: List of pycrypto X509 objects.


def verify_sans(amazon_cert: crypto.X509) -> bool:

cert_extentions = [amazon_cert.get_extension(i) for i in range(amazon_cert.get_extension_count())]<EOL>subject_alt_names = '<STR_LIT>'<EOL>for extention in cert_extentions:<EOL><INDENT>if '<STR_LIT>' in str(extention.get_short_name()):<EOL><INDENT>subject_alt_names = extention.__str__()<EOL>break<EOL><DEDENT><DEDENT>result = '<STR_LIT>' in subject_alt_names<EOL>return result<EOL>

Verifies Subject Alternative Names (SANs) for Amazon certificate.

    Args:
        amazon_cert: Pycrypto X509 Amazon certificate.

    Returns:
        result: True if verification was successful, False if not.


def verify_certs_chain(certs_chain: List[crypto.X509], amazon_cert: crypto.X509) -> bool:

store = crypto.X509Store()<EOL>for cert in certs_chain:<EOL><INDENT>store.add_cert(cert)<EOL><DEDENT>default_verify_paths = ssl.get_default_verify_paths()<EOL>default_verify_file = default_verify_paths.cafile<EOL>default_verify_file = Path(default_verify_file).resolve() if default_verify_file else None<EOL>default_verify_path = default_verify_paths.capath<EOL>default_verify_path = Path(default_verify_path).resolve() if default_verify_path else None<EOL>ca_files = [ca_file for ca_file in default_verify_path.iterdir()] if default_verify_path else []<EOL>if default_verify_file:<EOL><INDENT>ca_files.append(default_verify_file)<EOL><DEDENT>for ca_file in ca_files:<EOL><INDENT>ca_file: Path<EOL>if ca_file.is_file():<EOL><INDENT>with ca_file.open('<STR_LIT:r>', encoding='<STR_LIT:ascii>') as crt_f:<EOL><INDENT>ca_certs_txt = crt_f.read()<EOL>ca_certs = extract_certs(ca_certs_txt)<EOL>for cert in ca_certs:<EOL><INDENT>store.add_cert(cert)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>ssl_context = ssl.create_default_context()<EOL>der_certs = ssl_context.get_ca_certs(binary_form=True)<EOL>pem_certs = '<STR_LIT:\n>'.join([ssl.DER_cert_to_PEM_cert(der_cert) for der_cert in der_certs])<EOL>ca_certs = extract_certs(pem_certs)<EOL>for ca_cert in ca_certs:<EOL><INDENT>store.add_cert(ca_cert)<EOL><DEDENT>store_context = crypto.X509StoreContext(store, amazon_cert)<EOL>try:<EOL><INDENT>store_context.verify_certificate()<EOL>result = True<EOL><DEDENT>except crypto.X509StoreContextError:<EOL><INDENT>result = False<EOL><DEDENT>return result<EOL>

Verifies if Amazon and additional certificates creates chain of trust to a root CA.

    Args:
        certs_chain: List of pycrypto X509 intermediate certificates from signature chain URL.
        amazon_cert: Pycrypto X509 Amazon certificate.

    Returns:
        result: True if verification was successful, False if not.


def verify_signature(amazon_cert: crypto.X509, signature: str, request_body: bytes) -> bool:

signature = base64.b64decode(signature)<EOL>try:<EOL><INDENT>crypto.verify(amazon_cert, signature, request_body, '<STR_LIT>')<EOL>result = True<EOL><DEDENT>except crypto.Error:<EOL><INDENT>result = False<EOL><DEDENT>return result<EOL>

Verifies Alexa request signature.

    Args:
        amazon_cert: Pycrypto X509 Amazon certificate.
        signature: Base64 decoded Alexa request signature from Signature HTTP header.
        request_body: full HTTPS request body
    Returns:
        result: True if verification was successful, False if not.


def verify_cert(signature_chain_url: str) -> Optional[crypto.X509]:

try:<EOL><INDENT>certs_chain_get = requests.get(signature_chain_url)<EOL><DEDENT>except requests.exceptions.ConnectionError as e:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return None<EOL><DEDENT>certs_chain_txt = certs_chain_get.text<EOL>certs_chain = extract_certs(certs_chain_txt)<EOL>amazon_cert: crypto.X509 = certs_chain.pop(<NUM_LIT:0>)<EOL>sc_url_verification = verify_sc_url(signature_chain_url)<EOL>if not sc_url_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>expired_verification = not amazon_cert.has_expired()<EOL>if not expired_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>sans_verification = verify_sans(amazon_cert)<EOL>if not sans_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>chain_verification = verify_certs_chain(certs_chain, amazon_cert)<EOL>if not chain_verification:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL><DEDENT>result = (sc_url_verification and expired_verification and sans_verification and chain_verification)<EOL>return amazon_cert if result else None<EOL>

Conducts series of Alexa SSL certificate verifications against Amazon Alexa requirements.

    Args:
        signature_chain_url: Signature certificate URL from SignatureCertChainUrl HTTP header.
    Returns:
        result: Amazon certificate if verification was successful, None if not.


self.timer = Timer(self.config['<STR_LIT>'], self.self_destruct_callback)<EOL>self.timer.start()<EOL>


self.timer.cancel()<EOL>self._start_timer()<EOL>


def handle_request(self, request: dict) -> dict:

request_type = request['<STR_LIT>']['<STR_LIT:type>']<EOL>request_id = request['<STR_LIT>']['<STR_LIT>']<EOL>log.debug(f'<STR_LIT>')<EOL>if request_type in self.handled_requests.keys():<EOL><INDENT>response: dict = self.handled_requests[request_type](request)<EOL><DEDENT>else:<EOL><INDENT>response: dict = self.handled_requests['<STR_LIT>'](request)<EOL>log.warning(f'<STR_LIT>')<EOL><DEDENT>self._rearm_self_destruct()<EOL>return response<EOL>

Routes Alexa requests to appropriate handlers.

        Args:
            request: Alexa request.
        Returns:
            response: Response conforming Alexa response specification.


if self.stateful:<EOL><INDENT>utterance = [[utterance], [self.key]]<EOL><DEDENT>else:<EOL><INDENT>utterance = [[utterance]]<EOL><DEDENT>agent_response: list = self.agent(*utterance)<EOL>return agent_response<EOL>

Infers DeepPavlov agent with raw user input extracted from Alexa request.

        Args:
            utterance: Raw user input extracted from Alexa request.
        Returns:
            response: DeepPavlov agent response.


def _generate_response(self, response: dict, request: dict) -> dict:

response_template = deepcopy(self.response_template)<EOL>response_template['<STR_LIT>']['<STR_LIT>'] = request['<STR_LIT>']['<STR_LIT>']<EOL>for key, value in response_template.items():<EOL><INDENT>if key not in response.keys():<EOL><INDENT>response[key] = value<EOL><DEDENT><DEDENT>return response<EOL>

Populates generated response with additional data conforming Alexa response specification.

        Args:
            response: Raw user input extracted from Alexa request.
            request: Alexa request.
        Returns:
            response: Response conforming Alexa response specification.


def _handle_intent(self, request: dict) -> dict:

intent_name = self.config['<STR_LIT>']<EOL>slot_name = self.config['<STR_LIT>']<EOL>request_id = request['<STR_LIT>']['<STR_LIT>']<EOL>request_intent: dict = request['<STR_LIT>']['<STR_LIT>']<EOL>if intent_name != request_intent['<STR_LIT:name>']:<EOL><INDENT>log.error(f"<STR_LIT>")<EOL>return {'<STR_LIT:error>': '<STR_LIT>'}<EOL><DEDENT>if slot_name not in request_intent['<STR_LIT>'].keys():<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return {'<STR_LIT:error>': '<STR_LIT>'}<EOL><DEDENT>utterance = request_intent['<STR_LIT>'][slot_name]['<STR_LIT:value>']<EOL>agent_response = self._act(utterance)<EOL>if not agent_response:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return {'<STR_LIT:error>': '<STR_LIT>'}<EOL><DEDENT>prediction: RichMessage = agent_response[<NUM_LIT:0>]<EOL>prediction: list = prediction.alexa()<EOL>if not prediction:<EOL><INDENT>log.error(f'<STR_LIT>')<EOL>return {'<STR_LIT:error>': '<STR_LIT>'}<EOL><DEDENT>response = self._generate_response(prediction[<NUM_LIT:0>], request)<EOL>return response<EOL>

Handles IntentRequest Alexa request.

        Args:
            request: Alexa request.
        Returns:
            response: "response" part of response dict conforming Alexa specification.


def _handle_launch(self, request: dict) -> dict:

response = {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT:text>': self.config['<STR_LIT>']<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT:content>': self.config['<STR_LIT>']<EOL>}<EOL>}<EOL>}<EOL>response = self._generate_response(response, request)<EOL>return response<EOL>

Handles LaunchRequest Alexa request.

        Args:
            request: Alexa request.
        Returns:
            response: "response" part of response dict conforming Alexa specification.


def _handle_end(self, request: dict) -> dict:

response = {}<EOL>self.self_destruct_callback()<EOL>return response<EOL>

Handles SessionEndedRequest Alexa request and deletes Conversation instance.

        Args:
            request: Alexa request.
        Returns:
            response: Dummy empty response dict.


def _handle_unsupported(self, request: dict) -> dict:

Handles all unsupported types of Alexa requests. Returns standard message.

        Args:
            request: Alexa request.
        Returns:
            response: "response" part of response dict conforming Alexa specification.


def run_alexa_default_agent(model_config: Union[str, Path, dict],<EOL>multi_instance: bool = False,<EOL>stateful: bool = False,<EOL>port: Optional[int] = None,<EOL>https: bool = False,<EOL>ssl_key: str = None,<EOL>ssl_cert: str = None,<EOL>default_skill_wrap: bool = True) -> None:

def get_default_agent() -> DefaultAgent:<EOL><INDENT>model = build_model(model_config)<EOL>skill = DefaultStatelessSkill(model) if default_skill_wrap else model<EOL>agent = DefaultAgent([skill], skills_processor=DefaultRichContentWrapper())<EOL>return agent<EOL><DEDENT>run_alexa_server(agent_generator=get_default_agent,<EOL>multi_instance=multi_instance,<EOL>stateful=stateful,<EOL>port=port,<EOL>https=https,<EOL>ssl_key=ssl_key,<EOL>ssl_cert=ssl_cert)<EOL>

Creates Alexa agents factory and initiates Alexa web service.

    Wrapper around run_alexa_server. Allows raise Alexa web service with
    DeepPavlov config in backend.

    Args:
        model_config: DeepPavlov config path.
        multi_instance: Multi instance mode flag.
        stateful: Stateful mode flag.
        port: Flask web service port.
        https: Flag for running Alexa skill service in https mode.
        ssl_key: SSL key file path.
        ssl_cert: SSL certificate file path.
        default_skill_wrap: Wrap with default skill flag.


def run_alexa_server(agent_generator: callable,<EOL>multi_instance: bool = False,<EOL>stateful: bool = False,<EOL>port: Optional[int] = None,<EOL>https: bool = False,<EOL>ssl_key: str = None,<EOL>ssl_cert: str = None) -> None:

server_config_path = Path(get_settings_path(), SERVER_CONFIG_FILENAME).resolve()<EOL>server_params = read_json(server_config_path)<EOL>host = server_params['<STR_LIT>']['<STR_LIT:host>']<EOL>port = port or server_params['<STR_LIT>']['<STR_LIT:port>']<EOL>alexa_server_params = server_params['<STR_LIT>']<EOL>alexa_server_params['<STR_LIT>'] = multi_instance or server_params['<STR_LIT>']['<STR_LIT>']<EOL>alexa_server_params['<STR_LIT>'] = stateful or server_params['<STR_LIT>']['<STR_LIT>']<EOL>alexa_server_params['<STR_LIT>'] = AMAZON_CERTIFICATE_LIFETIME<EOL>if https:<EOL><INDENT>ssh_key_path = Path(ssl_key or server_params['<STR_LIT>']).resolve()<EOL>if not ssh_key_path.is_file():<EOL><INDENT>e = FileNotFoundError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>log.error(e)<EOL>raise e<EOL><DEDENT>ssh_cert_path = Path(ssl_cert or server_params['<STR_LIT>']).resolve()<EOL>if not ssh_cert_path.is_file():<EOL><INDENT>e = FileNotFoundError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>log.error(e)<EOL>raise e<EOL><DEDENT>ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)<EOL>ssl_context.load_cert_chain(ssh_cert_path, ssh_key_path)<EOL><DEDENT>else:<EOL><INDENT>ssl_context = None<EOL><DEDENT>input_q = Queue()<EOL>output_q = Queue()<EOL>bot = Bot(agent_generator, alexa_server_params, input_q, output_q)<EOL>bot.start()<EOL>endpoint_description = {<EOL>'<STR_LIT:description>': '<STR_LIT>',<EOL>'<STR_LIT>': [<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT:true>',<EOL>'<STR_LIT:type>': '<STR_LIT:string>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>},<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT:true>',<EOL>'<STR_LIT:type>': '<STR_LIT:string>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>},<EOL>{<EOL>'<STR_LIT:name>': '<STR_LIT:data>',<EOL>'<STR_LIT>': '<STR_LIT:body>',<EOL>'<STR_LIT>': '<STR_LIT:true>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:version>': '<STR_LIT:1.0>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>},<EOL>'<STR_LIT:user>': {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>}<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>},<EOL>'<STR_LIT:user>': {<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': {}<EOL>},<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>'<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': [<EOL>{<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': False,<EOL>'<STR_LIT>': False<EOL>}<EOL>],<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': <NUM_LIT>,<EOL>'<STR_LIT>': [<EOL>'<STR_LIT>'<EOL>]<EOL>}<EOL>},<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:type>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': {<EOL>'<STR_LIT:name>': '<STR_LIT>',<EOL>'<STR_LIT:value>': '<STR_LIT>',<EOL>'<STR_LIT>': {<EOL>'<STR_LIT>': [<EOL>{<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:status>': {<EOL>'<STR_LIT:code>': '<STR_LIT>'<EOL>}<EOL>}<EOL>]<EOL>},<EOL>'<STR_LIT>': '<STR_LIT>',<EOL>'<STR_LIT:source>': '<STR_LIT>'<EOL>}<EOL>}<EOL>}<EOL>}<EOL>}<EOL>}<EOL>],<EOL>'<STR_LIT>': {<EOL>"<STR_LIT>": {<EOL>"<STR_LIT:description>": "<STR_LIT>"<EOL>}<EOL>}<EOL>}<EOL>@app.route('<STR_LIT:/>')<EOL>def index():<EOL><INDENT>return redirect('<STR_LIT>')<EOL><DEDENT>@app.route('<STR_LIT>', methods=['<STR_LIT:POST>'])<EOL>@swag_from(endpoint_description)<EOL>def handle_request():<EOL><INDENT>request_body: bytes = request.get_data()<EOL>signature_chain_url: str = request.headers.get('<STR_LIT>')<EOL>signature: str = request.headers.get('<STR_LIT>')<EOL>alexa_request: dict = request.get_json()<EOL>request_dict = {<EOL>'<STR_LIT>': request_body,<EOL>'<STR_LIT>': signature_chain_url,<EOL>'<STR_LIT>': signature,<EOL>'<STR_LIT>': alexa_request<EOL>}<EOL>bot.input_queue.put(request_dict)<EOL>response: dict = bot.output_queue.get()<EOL>response_code = <NUM_LIT> if '<STR_LIT:error>' in response.keys() else <NUM_LIT:200><EOL>return jsonify(response), response_code<EOL><DEDENT>app.run(host=host, port=port, threaded=True, ssl_context=ssl_context)<EOL>

Initiates Flask web service with Alexa skill.

    Args:
        agent_generator: Callback Alexa agents factory.
        multi_instance: Multi instance mode flag.
        stateful: Stateful mode flag.
        port: Flask web service port.
        https: Flag for running Alexa skill service in https mode.
        ssl_key: SSL key file path.
        ssl_cert: SSL certificate file path.


def __init__(self, data: dict,<EOL>fields_to_merge: List[str] = None, merged_field: str = None,<EOL>field_to_split: str = None, split_fields: List[str] = None, split_proportions: List[float] = None,<EOL>seed: int = None, shuffle: bool = True, split_seed: int=None,<EOL>stratify: bool = None,<EOL>*args, **kwargs):

super().__init__(data, seed=seed, shuffle=shuffle)<EOL>if fields_to_merge is not None:<EOL><INDENT>if merged_field is not None:<EOL><INDENT>log.info("<STR_LIT>".format(fields_to_merge,<EOL>merged_field))<EOL>self._merge_data(fields_to_merge=fields_to_merge,<EOL>merged_field=merged_field)<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT><DEDENT>if field_to_split is not None:<EOL><INDENT>if split_fields is not None:<EOL><INDENT>log.info("<STR_LIT>".format(field_to_split,<EOL>split_fields))<EOL>self._split_data(field_to_split=field_to_split,<EOL>split_fields=split_fields,<EOL>split_proportions=[float(s) for s in<EOL>split_proportions],<EOL>split_seed=split_seed,<EOL>stratify=stratify)<EOL><DEDENT>else:<EOL><INDENT>raise IOError("<STR_LIT>")<EOL><DEDENT><DEDENT>

Initialize dataset using data from DatasetReader,
merges and splits fields according to the given parameters


def _split_data(self, field_to_split: str = None, split_fields: List[str] = None,<EOL>split_proportions: List[float] = None, split_seed: int=None, stratify: bool = None) -> bool:

if split_seed is None:<EOL><INDENT>split_seed = self.random.randint(<NUM_LIT:0>, <NUM_LIT>)<EOL><DEDENT>data_to_div = self.data[field_to_split].copy()<EOL>data_size = len(self.data[field_to_split])<EOL>for i in range(len(split_fields) - <NUM_LIT:1>):<EOL><INDENT>if stratify:<EOL><INDENT>stratify = [sample[<NUM_LIT:1>] for sample in data_to_div]<EOL><DEDENT>self.data[split_fields[i]], data_to_div = train_test_split(<EOL>data_to_div,<EOL>test_size=len(data_to_div) - int(data_size * split_proportions[i]),<EOL>random_state=split_seed,<EOL>stratify=stratify)<EOL>self.data[split_fields[-<NUM_LIT:1>]] = data_to_div<EOL><DEDENT>return True<EOL>

Split given field of dataset to the given list of fields with corresponding proportions

Args:
    field_to_split: field name (out of ``"train", "valid", "test"``) which to split
    split_fields: list of names (out of ``"train", "valid", "test"``) of fields to which split
    split_proportions: corresponding proportions
    split_seed: random seed for splitting dataset
    stratify: whether to use stratified split

Returns:
    None


def _merge_data(self, fields_to_merge: List[str] = None, merged_field: str = None) -> bool:

data = self.data.copy()<EOL>data[merged_field] = []<EOL>for name in fields_to_merge:<EOL><INDENT>data[merged_field] += self.data[name]<EOL><DEDENT>self.data = data<EOL>return True<EOL>

Merge given fields of dataset

Args:
    fields_to_merge: list of fields (out of ``"train", "valid", "test"``) to merge
    merged_field: name of field (out of ``"train", "valid", "test"``) to which save merged fields

Returns:
    None


def __init__(self, data: dict,<EOL>fields_to_merge: List[str] = None, merged_field: str = None,<EOL>field_to_split: str = None, split_fields: List[str] = None, split_proportions: List[float] = None,<EOL>seed: int = None, shuffle: bool = True,<EOL>*args, **kwargs):

super().__init__(data, fields_to_merge, merged_field,<EOL>field_to_split, split_fields, split_proportions,<EOL>seed=seed, shuffle=shuffle)<EOL>new_data = dict()<EOL>new_data['<STR_LIT:train>'] = []<EOL>new_data['<STR_LIT>'] = []<EOL>new_data['<STR_LIT:test>'] = []<EOL>for field in ['<STR_LIT:train>', '<STR_LIT>', '<STR_LIT:test>']:<EOL><INDENT>for turn in self.data[field]:<EOL><INDENT>reply = turn[<NUM_LIT:0>]<EOL>curr_intents = []<EOL>if reply['<STR_LIT>']:<EOL><INDENT>for intent in reply['<STR_LIT>']:<EOL><INDENT>for slot in intent['<STR_LIT>']:<EOL><INDENT>if slot[<NUM_LIT:0>] == '<STR_LIT>':<EOL><INDENT>curr_intents.append(intent['<STR_LIT>'] + '<STR_LIT:_>' + slot[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>curr_intents.append(intent['<STR_LIT>'] + '<STR_LIT:_>' + slot[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>if len(intent['<STR_LIT>']) == <NUM_LIT:0>:<EOL><INDENT>curr_intents.append(intent['<STR_LIT>'])<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if reply['<STR_LIT:text>']:<EOL><INDENT>curr_intents.append('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>continue<EOL><DEDENT><DEDENT>new_data[field].append((reply['<STR_LIT:text>'], curr_intents))<EOL><DEDENT><DEDENT>self.data = new_data<EOL>


@overrides<EOL><INDENT>def get_doc_ids(self) -> List[Any]:<DEDENT>

cursor = self.connect.cursor()<EOL>cursor.execute('<STR_LIT>'.format(self.db_name))<EOL>ids = [ids[<NUM_LIT:0>] for ids in cursor.fetchall()]<EOL>cursor.close()<EOL>return ids<EOL>

Get document ids.

        Returns:
            document ids


cursor = self.connect.cursor()<EOL>cursor.execute("<STR_LIT>")<EOL>assert cursor.arraysize == <NUM_LIT:1><EOL>name = cursor.fetchone()[<NUM_LIT:0>]<EOL>cursor.close()<EOL>return name<EOL>

Get DB name.

        Returns:
            DB name


def map_doc2idx(self) -> Dict[int, Any]:

doc2idx = {doc_id: i for i, doc_id in enumerate(self.doc_ids)}<EOL>logger.info(<EOL>"<STR_LIT>".format(len(doc2idx)))<EOL>return doc2idx<EOL>

Map DB ids to integer ids.

        Returns:
            a dictionary of document titles and correspondent integer indices


@overrides<EOL><INDENT>def get_doc_content(self, doc_id: Any) -> Optional[str]:<DEDENT>

cursor = self.connect.cursor()<EOL>cursor.execute(<EOL>"<STR_LIT>".format(self.db_name),<EOL>(doc_id,)<EOL>)<EOL>result = cursor.fetchone()<EOL>cursor.close()<EOL>return result if result is None else result[<NUM_LIT:0>]<EOL>

Get document content by id.

        Args:
            doc_id: a document id

        Returns:
            document content if success, else raise Exception


@overrides<EOL><INDENT>def gen_batches(self, batch_size: int, shuffle: bool = None)-> Generator[Tuple[List[str], List[int]], Any, None]:<DEDENT>

if shuffle is None:<EOL><INDENT>shuffle = self.shuffle<EOL><DEDENT>if shuffle:<EOL><INDENT>_doc_ids = self.random.sample(self.doc_ids, len(self.doc_ids))<EOL><DEDENT>else:<EOL><INDENT>_doc_ids = self.doc_ids<EOL><DEDENT>if batch_size > <NUM_LIT:0>:<EOL><INDENT>batches = [_doc_ids[i:i + batch_size] for i in<EOL>range(<NUM_LIT:0>, len(_doc_ids), batch_size)]<EOL><DEDENT>else:<EOL><INDENT>batches = [_doc_ids]<EOL><DEDENT>for i, doc_ids in enumerate(batches):<EOL><INDENT>docs = [self.get_doc_content(doc_id) for doc_id in doc_ids]<EOL>doc_nums = [self.doc2index[doc_id] for doc_id in doc_ids]<EOL>yield docs, zip(doc_ids, doc_nums)<EOL><DEDENT>

Gen batches of documents.

        Args:
            batch_size: a number of samples in a single batch
            shuffle: whether to shuffle data during batching

        Yields:
            generated tuple of documents and their ids


doc_ids = list(self.doc_ids)<EOL>docs = [self.get_doc_content(doc_id) for doc_id in doc_ids]<EOL>doc_nums = [self.doc2index[doc_id] for doc_id in doc_ids]<EOL>return docs, zip(doc_ids, doc_nums)<EOL>


@staticmethod<EOL><INDENT>def _extract_cqas(data: Dict[str, Any]) -> List[Tuple[Tuple[str, str], Tuple[List[str], List[int]]]]:<DEDENT>

cqas = []<EOL>if data:<EOL><INDENT>for article in data['<STR_LIT:data>']:<EOL><INDENT>for par in article['<STR_LIT>']:<EOL><INDENT>context = par['<STR_LIT>']<EOL>for qa in par['<STR_LIT>']:<EOL><INDENT>q = qa['<STR_LIT>']<EOL>ans_text = []<EOL>ans_start = []<EOL>for answer in qa['<STR_LIT>']:<EOL><INDENT>ans_text.append(answer['<STR_LIT:text>'])<EOL>ans_start.append(answer['<STR_LIT>'])<EOL><DEDENT>cqas.append(((context, q), (ans_text, ans_start)))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return cqas<EOL>

Extracts context, question, answer, answer_start from SQuAD data

        Args:
            data: data in squad format

        Returns:
            list of (context, question), (answer_text, answer_start)
            answer text and answer_start are lists


def preprocess_data(data: List[Tuple[List[str], List[str]]], to_lower: bool = True,<EOL>append_case: str = "<STR_LIT>") -> List[Tuple[List[Tuple[str]], List[str]]]:

new_data = []<EOL>for words, tags in data:<EOL><INDENT>new_words = [process_word(word, to_lower=to_lower, append_case=append_case)<EOL>for word in words]<EOL>new_tags = tags<EOL>new_data.append((new_words, new_tags))<EOL><DEDENT>return new_data<EOL>

Processes all words in data using
    :func:`~deeppavlov.dataset_iterators.morphotagger_iterator.process_word`.

    Args:
        data: a list of pairs (words, tags), each pair corresponds to a single sentence
        to_lower: whether to lowercase
        append_case: whether to add case mark

    Returns:
        a list of preprocessed sentences


if len(self.valid) == <NUM_LIT:0>:<EOL><INDENT>if self.shuffle:<EOL><INDENT>random.shuffle(self.train)<EOL><DEDENT>L = int(len(self.train) * (<NUM_LIT:1.0> - self.validation_split))<EOL>self.train, self.valid = self.train[:L], self.valid[L:]<EOL><DEDENT>elif self.min_train_fraction > <NUM_LIT:0.0>:<EOL><INDENT>train_length = len(self.train)<EOL>valid_length = len(self.valid)<EOL>gap = int(self.min_train_fraction * (train_length + valid_length)) - train_length<EOL>if gap > <NUM_LIT:0>:<EOL><INDENT>self.train.extend(self.valid[:gap])<EOL>self.valid = self.valid[gap:]<EOL><DEDENT><DEDENT>return<EOL>

Splits the `train` part to `train` and `valid`, if no `valid` part is specified.
Moves deficient data from `valid` to `train` if both parts are given,
but `train` subset is too small.


def gen_batches(self, batch_size: int, data_type: str = '<STR_LIT:train>',<EOL>shuffle: bool = None, return_indexes: bool = False) -> Iterator[tuple]:

if shuffle is None:<EOL><INDENT>shuffle = self.shuffle<EOL><DEDENT>data = self.data[data_type]<EOL>if shuffle:<EOL><INDENT>random.shuffle(data)<EOL><DEDENT>lengths = [len(x[<NUM_LIT:0>]) for x in data]<EOL>indexes = np.argsort(lengths)<EOL>L = len(data)<EOL>if batch_size < <NUM_LIT:0>:<EOL><INDENT>batch_size = L<EOL><DEDENT>for start in range(<NUM_LIT:0>, L, batch_size):<EOL><INDENT>indexes_to_yield = indexes[start:start+batch_size]<EOL>data_to_yield = tuple(list(x) for x in zip(*([data[i] for i in indexes_to_yield])))<EOL>if return_indexes:<EOL><INDENT>yield indexes_to_yield, data_to_yield<EOL><DEDENT>else:<EOL><INDENT>yield data_to_yield<EOL><DEDENT><DEDENT>

Generate batches of inputs and expected output to train neural networks

        Args:
            batch_size: number of samples in batch
            data_type: can be either 'train', 'test', or 'valid'
            shuffle: whether to shuffle dataset before batching
            return_indexes: whether to return indexes of batch elements in initial dataset

        Yields:
            a tuple of a batch of inputs and a batch of expected outputs.
            If `return_indexes` is True, also yields indexes of batch elements.


def gen_batches(self, batch_size: int, data_type: str = "<STR_LIT:train>", shuffle: bool = True)->Tuple[List[List[Tuple[int, int]]], List[int]]:

data = self.data[data_type]<EOL>if self.random_batches and self.batches_per_epoch is not None and data_type == "<STR_LIT:train>":<EOL><INDENT>num_steps = self.batches_per_epoch<EOL>if batch_size > len(data):<EOL><INDENT>batch_size = len(data)<EOL>log.warning("<STR_LIT>")<EOL><DEDENT><DEDENT>else:<EOL><INDENT>num_steps = len(data) // batch_size<EOL><DEDENT>if data_type == "<STR_LIT:train>":<EOL><INDENT>if shuffle:<EOL><INDENT>self.random.shuffle(data)<EOL><DEDENT>for i in range(num_steps):<EOL><INDENT>if self.random_batches:<EOL><INDENT>context_response_data = self.random.sample(data, k=batch_size)<EOL><DEDENT>else:<EOL><INDENT>context_response_data = data[i * batch_size:(i + <NUM_LIT:1>) * batch_size]<EOL><DEDENT>yield tuple(zip(*context_response_data))<EOL><DEDENT><DEDENT>if data_type in ["<STR_LIT>", "<STR_LIT:test>"]:<EOL><INDENT>for i in range(num_steps + <NUM_LIT:1>):<EOL><INDENT>context_response_data = data[i * batch_size:(i + <NUM_LIT:1>) * batch_size]<EOL>if context_response_data != []:<EOL><INDENT>yield tuple(zip(*context_response_data))<EOL><DEDENT><DEDENT><DEDENT>

Generate batches of inputs and expected outputs to train neural networks.

        Args:
            batch_size: number of samples in batch
            data_type: can be either 'train', 'test', or 'valid'
            shuffle: whether to shuffle dataset before batching

        Yields:
            A tuple of a batch of inputs and a batch of expected outputs.

            Inputs and expected outputs have different structure and meaning
            depending on class attributes values and ``data_type``.


def get_instances(self, data_type: str = '<STR_LIT:train>') -> Tuple[List[List[str]], List[List[str]]]:

if data_type == '<STR_LIT:train>':<EOL><INDENT>samples = [self.data[data_type][i] for i in self.marked_nums]<EOL><DEDENT>else:<EOL><INDENT>samples = self.data[data_type][:]<EOL><DEDENT>x, y = list(zip(*samples))<EOL>return x, y<EOL>

Get all data for a selected data type

        Args:
            data_type (str): can be either ``'train'``, ``'test'``, ``'valid'`` or ``'all'``

        Returns:
             a tuple of all inputs for a data type and all expected outputs for a data type


def __call__(self, doc_ids: Optional[List[List[Any]]] = None, *args, **kwargs) -> List[Union[str, List[str]]]:

all_contents = []<EOL>if not doc_ids:<EOL><INDENT>logger.warn('<STR_LIT>')<EOL>doc_ids = [self.get_doc_ids()]<EOL><DEDENT>for ids in doc_ids:<EOL><INDENT>contents = [self.get_doc_content(doc_id) for doc_id in ids]<EOL>if self.join_docs:<EOL><INDENT>contents = '<STR_LIT:U+0020>'.join(contents)<EOL><DEDENT>all_contents.append(contents)<EOL><DEDENT>return all_contents<EOL>

Get the contents of files, stacked by space or as they are.

        Args:
            doc_ids: a batch of lists of ids to get contents for

        Returns:
            a list of contents / list of lists of contents


args = parser.parse_args()<EOL>path = get_settings_path()<EOL>if args.default:<EOL><INDENT>if populate_settings_dir(force=True):<EOL><INDENT>print(f'<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>print(f'<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print(f'<STR_LIT>')<EOL><DEDENT>

DeepPavlov console configuration utility.


if cycle:<EOL><INDENT>p.text('<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>with p.group(<NUM_LIT:7>, '<STR_LIT>', '<STR_LIT:)>'):<EOL><INDENT>p.pretty(self._asdict())<EOL><DEDENT><DEDENT>

method that defines ``Struct``'s pretty printing rules for iPython

        Args:
            p (IPython.lib.pretty.RepresentationPrinter): pretty printer object
            cycle (bool): is ``True`` if pretty detected a cycle


reqs_path = os.path.join(__location__, '<STR_LIT>')<EOL>with open(reqs_path, encoding='<STR_LIT:utf8>') as f:<EOL><INDENT>reqs = [line.strip() for line in f if not line.strip().startswith('<STR_LIT:#>')]<EOL><DEDENT>names = []<EOL>links = []<EOL>for req in reqs:<EOL><INDENT>if '<STR_LIT>' in req:<EOL><INDENT>links.append(req)<EOL><DEDENT>else:<EOL><INDENT>names.append(req)<EOL><DEDENT><DEDENT>return {'<STR_LIT>': names, '<STR_LIT>': links}<EOL>

parses requirements from requirements.txt


kwargs = dict((k.lower(), kwargs[k].lower()) for k in kwargs)<EOL>if '<STR_LIT:type>' in kwargs and (<EOL>kwargs['<STR_LIT:type>'] != '<STR_LIT>' and kwargs['<STR_LIT:type>'] != '<STR_LIT>'):<EOL><INDENT>del(kwargs['<STR_LIT:type>'])<EOL><DEDENT>if '<STR_LIT>' in kwargs and (<EOL>kwargs['<STR_LIT>'] != '<STR_LIT>' and kwargs['<STR_LIT>'] != '<STR_LIT:all>'):<EOL><INDENT>del(kwargs['<STR_LIT>'])<EOL><DEDENT>if "<STR_LIT>" in kwargs or "<STR_LIT>" in kwargs:<EOL><INDENT>kwargs["<STR_LIT>"] = "<STR_LIT>"<EOL><DEDENT>url = "<STR_LIT>".format(self.API, urlencode(kwargs))<EOL>data = self._get_resource(url)<EOL>groups = []<EOL>for datum in data.get('<STR_LIT:data>', []):<EOL><INDENT>group = GroupReference()<EOL>group.uwregid = datum.get('<STR_LIT>')<EOL>group.display_name = datum.get('<STR_LIT>')<EOL>group.name = datum.get('<STR_LIT:id>')<EOL>group.url = datum.get('<STR_LIT:url>')<EOL>groups.append(group)<EOL><DEDENT>return groups<EOL>

Returns a list of restclients.GroupReference objects matching the
passed parameters. Valid parameters are:
    name: parts_of_name
        name may include the wild-card (*) character.
    stem: group_stem
    member: member netid
    owner: admin netid
    instructor: instructor netid
        stem="course" will be set when this parameter is passed.
    student: student netid
        stem="course" will be set when this parameter is passed.
    affiliate: affiliate_name
    type: search_type
        Values are 'direct' to search for direct membership and
        'effective' to search for effective memberships. Default is
        direct membership.
    scope: search_scope
        Values are 'one' to limit results to one level of stem name
        and 'all' to return all groups.


self._valid_group_id(group_id)<EOL>url = "<STR_LIT>".format(self.API, group_id)<EOL>data = self._get_resource(url)<EOL>return self._group_from_json(data.get("<STR_LIT:data>"))<EOL>

Returns a restclients.Group object for the group identified by the
passed group ID.


self._valid_group_id(group.id)<EOL>body = {"<STR_LIT:data>": group.json_data()}<EOL>url = "<STR_LIT>".format(self.API, group.name)<EOL>data = self._put_resource(url, headers={}, body=body)<EOL>return self._group_from_json(data.get("<STR_LIT:data>"))<EOL>

Creates a group from the passed restclients.Group object.


self._valid_group_id(group.id)<EOL>body = {"<STR_LIT:data>": group.json_data()}<EOL>headers = {"<STR_LIT>": "<STR_LIT:*>"}<EOL>url = "<STR_LIT>".format(self.API, group.name)<EOL>data = self._put_resource(url, headers, body)<EOL>return self._group_from_json(data.get("<STR_LIT:data>"))<EOL>

Updates a group from the passed restclients.Group object.


self._valid_group_id(group_id)<EOL>url = "<STR_LIT>".format(self.API, group_id)<EOL>self._delete_resource(url)<EOL>return True<EOL>

Deletes the group identified by the passed group ID.


self._valid_group_id(group_id)<EOL>url = "<STR_LIT>".format(self.API, group_id)<EOL>data = self._get_resource(url)<EOL>members = []<EOL>for datum in data.get("<STR_LIT:data>"):<EOL><INDENT>members.append(self._group_member_from_json(datum))<EOL><DEDENT>return members<EOL>

Returns a list of restclients.GroupMember objects for the group
identified by the passed group ID.


def update_members(self, group_id, members):

self._valid_group_id(group_id)<EOL>body = {"<STR_LIT:data>": [m.json_data() for m in members]}<EOL>headers = {"<STR_LIT>": "<STR_LIT:*>"}<EOL>url = "<STR_LIT>".format(self.API, group_id)<EOL>data = self._put_resource(url, headers, body)<EOL>errors = data.get("<STR_LIT>", [])<EOL>if len(errors):<EOL><INDENT>return errors[<NUM_LIT:0>].get("<STR_LIT>", [])<EOL><DEDENT>return []<EOL>

Updates the membership of the group represented by the passed group id.
Returns a list of members not found.


Returns a list of effective restclients.GroupMember objects for the
group identified by the passed group ID.


self._valid_group_id(group_id)<EOL>url = "<STR_LIT>".format(self.API,<EOL>group_id)<EOL>data = self._get_resource(url)<EOL>count = data.get("<STR_LIT:data>").get("<STR_LIT:count>")<EOL>return int(count)<EOL>

Returns a count of effective members for the group identified by the
passed group ID.


def is_effective_member(self, group_id, netid):

self._valid_group_id(group_id)<EOL>netid = re.sub('<STR_LIT>', '<STR_LIT>', netid)<EOL>url = "<STR_LIT>".format(self.API,<EOL>group_id,<EOL>netid)<EOL>try:<EOL><INDENT>data = self._get_resource(url)<EOL>return True  <EOL><DEDENT>except DataFailureException as ex:<EOL><INDENT>if ex.status == <NUM_LIT>:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Returns True if the netid is in the group, False otherwise.


msg = self.format(record)<EOL>if not isinstance(msg, dict):<EOL><INDENT>msg = json.loads(msg)<EOL><DEDENT>self.collection.insert(msg)<EOL>


return {<EOL>ARM32:   (KS_ARCH_ARM,     KS_MODE_ARM),<EOL>ARM64:   (KS_ARCH_ARM64,   KS_MODE_LITTLE_ENDIAN),<EOL>ARM_TB:  (KS_ARCH_ARM,     KS_MODE_THUMB),<EOL>HEXAGON: (KS_ARCH_HEXAGON, KS_MODE_BIG_ENDIAN),<EOL>MIPS32:  (KS_ARCH_MIPS,    KS_MODE_MIPS32),<EOL>MIPS64:  (KS_ARCH_MIPS,    KS_MODE_MIPS64),<EOL>PPC32:   (KS_ARCH_PPC,     KS_MODE_PPC32),<EOL>PPC64:   (KS_ARCH_PPC,     KS_MODE_PPC64),<EOL>SPARC32: (KS_ARCH_SPARC,   KS_MODE_SPARC32),<EOL>SPARC64: (KS_ARCH_SPARC,   KS_MODE_SPARC64),<EOL>SYSTEMZ: (KS_ARCH_SYSTEMZ, KS_MODE_BIG_ENDIAN),<EOL>X86_16:  (KS_ARCH_X86,     KS_MODE_16),<EOL>X86_32:  (KS_ARCH_X86,     KS_MODE_32),<EOL>X86_64:  (KS_ARCH_X86,     KS_MODE_64),<EOL>}<EOL>

Initialize the dictionary of architectures for assembling via keystone


return {<EOL>ARM32:   (CS_ARCH_ARM,   CS_MODE_ARM),<EOL>ARM64:   (CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN),<EOL>ARM_TB:  (CS_ARCH_ARM,   CS_MODE_THUMB),<EOL>MIPS32:  (CS_ARCH_MIPS,  CS_MODE_MIPS32),<EOL>MIPS64:  (CS_ARCH_MIPS,  CS_MODE_MIPS64),<EOL>SPARC32: (CS_ARCH_SPARC, CS_MODE_BIG_ENDIAN),<EOL>SPARC64: (CS_ARCH_SPARC, CS_MODE_V9),<EOL>SYSTEMZ: (CS_ARCH_SYSZ,  CS_MODE_BIG_ENDIAN),<EOL>X86_16:  (CS_ARCH_X86,   CS_MODE_16),<EOL>X86_32:  (CS_ARCH_X86,   CS_MODE_32),<EOL>X86_64:  (CS_ARCH_X86,   CS_MODE_64),<EOL>}<EOL>

Initialize the dictionary of architectures for disassembling via capstone


def get_table(self, arch, pattern, colored=False, verbose=False):

rawtable = self.search(arch, pattern)<EOL>if len(rawtable) == <NUM_LIT:0>:<EOL><INDENT>return None<EOL><DEDENT>used_hd = self.__fetch_used_headers(rawtable, verbose)<EOL>table   = [self.__make_colored_row(used_hd, '<STR_LIT>', upper=True) if colored else used_hd]<EOL>for command in rawtable:<EOL><INDENT>cur_tb_field = []<EOL>for hd in used_hd:<EOL><INDENT>value = command[hd]<EOL>cur_tb_field.append(self.__make_colored_field(value, hd, verbose=verbose))<EOL><DEDENT>table.append(cur_tb_field)<EOL><DEDENT>return DoubleTable(table)<EOL>

This function is used in sys command (when user want to find a specific syscall)

:param Architecture for syscall table;
:param Searching pattern;
:param Flag for verbose output
:return Return a printable table of matched syscalls


if log.handlers:<EOL><INDENT>return<EOL><DEDENT>handler = logging.StreamHandler()<EOL>formatter = logging.Formatter(<EOL>'<STR_LIT>')<EOL>handler.setFormatter(formatter)<EOL>log.addHandler(handler)<EOL>

If there's no log configuration, set up a default handler.