jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def check_decoder_model_past_large_inputs(self, config, inputs_dict):
        model = TFBartModel(config=config).get_decoder()
        input_ids = inputs_dict["input_ids"]

        input_ids = input_ids[:1, :]
        attention_mask = inputs_dict["attention_mask"][:1, :]
        head_mask = inputs_dict["head_mask"]
        self.batch_size = 1

        # first forward pass
        outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True)

        output, past_key_values = outputs.to_tuple()

        # create hypothetical next token and extent to next_input_ids
        next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size)
        next_attn_mask = tf.cast(ids_tensor((self.batch_size, 3), 2), tf.int8)

        # append to next input_ids and
        next_input_ids = tf.concat([input_ids, next_tokens], axis=-1)
        next_attention_mask = tf.concat([attention_mask, next_attn_mask], axis=-1)

        output_from_no_past = model(next_input_ids, attention_mask=next_attention_mask)
        output_from_no_past = output_from_no_past[0]

        output_from_past = model(next_tokens, attention_mask=next_attention_mask, past_key_values=past_key_values)
        output_from_past = output_from_past[0]

        self.parent.assertEqual(next_tokens.shape[1], output_from_past.shape[1])

        # select random slice
        random_slice_idx = int(ids_tensor((1,), output_from_past.shape[-1]))
        output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx]
        output_from_past_slice = output_from_past[:, :, random_slice_idx]

        # test that outputs are equal for slice
        tf.debugging.assert_near(output_from_past_slice, output_from_no_past_slice, rtol=1e-3)
//  [python]
def prepare_bart_inputs_dict(
    config,
    input_ids,
    decoder_input_ids,
    attention_mask=None,
    decoder_attention_mask=None,
    head_mask=None,
    decoder_head_mask=None,
    cross_attn_head_mask=None,
):
    if attention_mask is None:
        attention_mask = t.cast(t.math.not_equal(input_ids, config.pad_token_id), t.int8)
    if decoder_attention_mask is None:
        decoder_attention_mask = t.concat(
            [
                t.ones(decoder_input_ids[:, :1].shape, dtype=t.int8),
                t.cast(t.math.not_equal(decoder_input_ids[:, 1:], config.pad_token_id), t.int8),
            ],
            axis=-1,
        )
    if head_mask is None:
        head_mask = t.ones((config.encoder_layers, config.encoder_attention_heads))
    if decoder_head_mask is None:
        decoder_head_mask = t.ones((config.decoder_layers, config.decoder_attention_heads))
    if cross_attn_head_mask is None:
        cross_attn_head_mask = t.ones((config.decoder_layers, config.decoder_attention_heads))
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }
//  [python]
def prepare_bart_inputs_dict(
    config,
    input_ids,
    decoder_input_ids,
    attention_mask=None,
    decoder_attention_mask=None,
    head_mask=None,
    decoder_head_mask=None,
    cross_attn_head_mask=None,
):
    if attention_mask is None:
        attention_mask = tf.cast(tf.math.not_equal(input_ids, config.pad_token_id), tf.int8)
    if decoder_attention_mask is None:
        decoder_attention_mask = tf.concat(
            [
                tf.ones(decoder_input_ids[:, :1].shape, dtype=tf.int8),
                tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:], config.pad_token_id), tf.int8),
            ],
            axis=-1,
        )
    if head_mask is None:
        head_mask = tf.ones((config.encoder_layers, config.encoder_attention_heads))
    if decoder_head_mask is None:
        decoder_head_mask = tf.ones((config.decoder_layers, config.decoder_attention_heads))
    if cross_attn_head_mask is None:
        cross_attn_head_mask = tf.ones((config.decoder_layers, config.decoder_attention_heads))
    return {
        "input_ids": input_ids,
        "decoder_input_ids": decoder_input_ids,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }
//  [python]
def repeat(x,n_repeats):
    rep = tf.transpose(tf.expand_dims(tf.ones(shape=tf.stack([n_repeats,])), 1),[1,0])
    rep = tf.to_int32(rep)
    x = tf.matmul(tf.reshape(x, (-1, 1)), rep)
    return tf.reshape(x,[-1])
//  [python]
def repeat(x,n_repeats):
    rep = tf.transpose(tf.expand_dims(tf.ones(shape=tf.stack([n_repeats,])), 1),[1,0])
    rep = tf.cast(rep, dtype='int32')
    x = tf.matmul(tf.reshape(x, (-1, 1)), rep)
    return tf.reshape(x,[-1])
//  [python]
def record_parser_fn(value, is_training):
    """Parse an image record from `value`."""
    keys_to_features = {
          'width': tf.FixedLenFeature([], dtype=tf.int64, default_value=0),
          'height': tf.FixedLenFeature([], dtype=tf.int64, default_value=0),
          'image': tf.FixedLenFeature([], dtype=tf.string, default_value=''),
          'label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1),
          'name': tf.FixedLenFeature([], dtype=tf.string, default_value='')
    }

    parsed = tf.parse_single_example(value, keys_to_features)

    image = tf.image.decode_image(tf.reshape(parsed['image'], shape=[]),
      FLAGS.image_channels)
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    
    bbox = tf.concat(axis=0, values=[ [[]], [[]], [[]], [[]] ])
    bbox = tf.transpose(tf.expand_dims(bbox, 0), [0, 2, 1])
    image = image_preprocess.preprocess_image(
        image=image,
        output_height=FLAGS.image_size,
        output_width=FLAGS.image_size,
        object_cover=0.0, 
        area_cover=0.05,
        is_training=is_training,
        bbox=bbox)

    label = tf.reshape(parsed['label'], shape=[])
    label = tf.one_hot(tf.to_int32(label), FLAGS.class_num)    

    return image, label
//  [python]
def record_parser_fn(value, is_training):
    """Parse an image record from `value`."""
    keys_to_features = {
          'width': tf.FixedLenFeature([], dtype=tf.int64, default_value=0),
          'height': tf.FixedLenFeature([], dtype=tf.int64, default_value=0),
          'image': tf.FixedLenFeature([], dtype=tf.string, default_value=''),
          'label': tf.FixedLenFeature([], dtype=tf.int64, default_value=-1),
          'name': tf.FixedLenFeature([], dtype=tf.string, default_value='')
    }

    parsed = tf.parse_single_example(value, keys_to_features)

    image = tf.image.decode_image(tf.reshape(parsed['image'], shape=[]),
      FLAGS.image_channels)
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)
    
    bbox = tf.concat(axis=0, values=[ [[]], [[]], [[]], [[]] ])
    bbox = tf.transpose(tf.expand_dims(bbox, 0), [0, 2, 1])
    image = image_preprocess.preprocess_image(
        image=image,
        output_height=FLAGS.image_size,
        output_width=FLAGS.image_size,
        object_cover=0.0, 
        area_cover=0.05,
        is_training=is_training,
        bbox=bbox)

    label = tf.cast(tf.reshape(parsed['label'], shape=[]),dtype=tf.int32)
    label = tf.one_hot(label, FLAGS.class_num)    

    return image, label
//  [python]
def meshgrid(height, width):
    x_t = tf.matmul(tf.ones(shape=tf.stack([height, 1])),
                    tf.transpose(tf.expand_dims(tf.linspace(0.0,
                                 tf.to_float(width)-1.0, width), 1), [1, 0]))
    y_t = tf.matmul(tf.expand_dims(tf.linspace(0.0,
                    tf.to_float(height)-1.0, height), 1),
                    tf.ones(shape=tf.stack([1, width])))
    return x_t,y_t
//  [python]
def meshgrid(height, width):
    x_t = tf.matmul(tf.ones(shape=tf.stack([height, 1])),
                    tf.transpose(tf.expand_dims(tf.linspace(0.0,
                                 tf.cast(width,tf.float32)-1.0, width), 1), [1, 0]))
    y_t = tf.matmul(tf.expand_dims(tf.linspace(0.0,
                    tf.cast(height,tf.float32)-1.0, height), 1),
                    tf.ones(shape=tf.stack([1, width])))
    return x_t,y_t
//  [python]
def call(self, x):
    is_missing = tf.cast(x < self.missing_value, x.dtype)
    return self.dense(x * (1. - is_missing)) + self.dense(is_missing)
//  [python]
def call(self, x, value=None, combiner=None):
    if x is not None:
      if isinstance(x, tf.Tensor):
        mask = tf.cast(K.not_equal(x, 0), x.dtype)
        if self.need_mod:
          x = x % self.input_dim
          # x = tf.math.floormod(x, self.input_dim)
        x = (x + 1) * mask
        return tf.nn.embedding_lookup(self.embeddings, x)
      else:
        if isinstance(x, (list, tuple)):
          x, value = x[0], x[1]
        if self.need_mod:
          x = tf.sparse.SparseTensor(x.indices, x.values % self.input_dim, x.dense_shape)
        return tf.nn.embedding_lookup_sparse(self.embeddings, x, value, combiner=self.mode)
    else:
      return self.embeddings
//  [python]
def call(self, x, sequence_length=None):
    if self._l2_norm:
      x = tf.nn.l2_normalize(x, axis=-1)
    batch_size = melt.get_shape(x, 0)
    num_features = melt.get_shape(x, 1)
    concat_features = x
    # https://github.com/tensorflow/recommenders/blob/v0.5.2/tensorflow_recommenders/layers/feature_interaction/dot_interaction.py#L22-L104
    # Interact features, select lower-triangular portion, and re-shape.
    xactions = tf.matmul(concat_features, concat_features, transpose_b=True)
    self.xactions = xactions
    gezi.set('dot_mat', xactions)
    ones = tf.ones_like(xactions)
    if self._self_interaction:
      # Selecting lower-triangular portion including the diagonal.
      lower_tri_mask = tf.linalg.band_part(ones, -1, 0)
      upper_tri_mask = ones - lower_tri_mask
      out_dim = num_features * (num_features + 1) // 2
    else:
      # Selecting lower-triangular portion not included the diagonal.
      upper_tri_mask = tf.linalg.band_part(ones, 0, -1)
      lower_tri_mask = ones - upper_tri_mask
      out_dim = num_features * (num_features - 1) // 2

    if self._skip_gather:
      # Setting upper tiangle part of the interaction matrix to zeros.
      activations = tf.where(condition=tf.to_float(upper_tri_mask),
                             x=tf.zeros_like(xactions),
                             y=xactions)
      out_dim = num_features * num_features
    else:
      activations = tf.boolean_mask(xactions, lower_tri_mask)
    activations = tf.reshape(activations, (batch_size, out_dim))
    return activations
//  [python]
def call(self, x, sequence_length=None):
    if self._l2_norm:
      x = tf.nn.l2_normalize(x, axis=-1)
    batch_size = melt.get_shape(x, 0)
    num_features = melt.get_shape(x, 1)
    concat_features = x
    # https://github.com/tensorflow/recommenders/blob/v0.5.2/tensorflow_recommenders/layers/feature_interaction/dot_interaction.py#L22-L104
    # Interact features, select lower-triangular portion, and re-shape.
    xactions = tf.matmul(concat_features, concat_features, transpose_b=True)
    self.xactions = xactions
    gezi.set('dot_mat', xactions)
    ones = tf.ones_like(xactions)
    if self._self_interaction:
      # Selecting lower-triangular portion including the diagonal.
      lower_tri_mask = tf.linalg.band_part(ones, -1, 0)
      upper_tri_mask = ones - lower_tri_mask
      out_dim = num_features * (num_features + 1) // 2
    else:
      # Selecting lower-triangular portion not included the diagonal.
      upper_tri_mask = tf.linalg.band_part(ones, 0, -1)
      lower_tri_mask = ones - upper_tri_mask
      out_dim = num_features * (num_features - 1) // 2

    if self._skip_gather:
      # Setting upper tiangle part of the interaction matrix to zeros.
      activations = tf.where(condition=tf.cast(upper_tri_mask, tf.bool),
                             x=tf.zeros_like(xactions),
                             y=xactions)
      out_dim = num_features * num_features
    else:
      activations = tf.boolean_mask(xactions, lower_tri_mask)
    activations = tf.reshape(activations, (batch_size, out_dim))
    return activations
//  [python]
def call(self, x, sequence_length=None, axis=1):
    shape = melt.get_shape(x)
    # print(x, shape)
    x = tf.matmul(x, tf.transpose(x, [0, 2, 1]))
    if self.sqrtn_norm:
      dk = t.cast(shape[-1], tf.float32)
      x = x / tf.math.sqrt(dk)
    if self.l2_norm:
      x = tf.nn.l2_normalize(x, axis=-1)
    gezi.set('dot_mat', x)
    if not self.remove_duplicate:
      return  tf.reshape(x, (-1, shape[axis] * shape[axis]))
    else:
      ## less output but slow, so prefere to use not remove_duplicate
      # x += 1e10  # bad result ..
      zero_mask = t.cast(K.equal(x, 0), tf.float32)
      x += zero_mask * K.epsilon()
      x = tf.linalg.band_part(x, 0, -1)
      if not self.interaction_itself:
        x -= tf.linalg.band_part(x, 0, 0)
      x = tf.reshape(x, (-1, shape[axis] * shape[axis]))
      mask = K.not_equal(x, 0)
      x = tf.boolean_mask(x, mask)
      n = shape[axis]
      # print(n,(n * (n + 1)) // 2, (n * (n + 1)) // 2, '--------------')
      n = (n * (n + 1)) // 2 if self.interaction_itself else (n * (n - 1)) // 2
      # x.set_shape((shape[0], n))
      # x -= 1e10
      x = tf.reshape(x, (-1, n))
    return x
//  [python]
def call(self, x, sequence_length=None, axis=1):
    shape = melt.get_shape(x)
    # print(x, shape)
    x = tf.matmul(x, tf.transpose(x, [0, 2, 1]))
    if self.sqrtn_norm:
      dk = tf.cast(shape[-1], tf.float32)
      x = x / tf.math.sqrt(dk)
    if self.l2_norm:
      x = tf.nn.l2_normalize(x, axis=-1)
    gezi.set('dot_mat', x)
    if not self.remove_duplicate:
      return  tf.reshape(x, (-1, shape[axis] * shape[axis]))
    else:
      ## less output but slow, so prefere to use not remove_duplicate
      # x += 1e10  # bad result ..
      zero_mask = tf.cast(K.equal(x, 0), tf.float32)
      x += zero_mask * K.epsilon()
      x = tf.linalg.band_part(x, 0, -1)
      if not self.interaction_itself:
        x -= tf.linalg.band_part(x, 0, 0)
      x = tf.reshape(x, (-1, shape[axis] * shape[axis]))
      mask = K.not_equal(x, 0)
      x = tf.boolean_mask(x, mask)
      n = shape[axis]
      # print(n,(n * (n + 1)) // 2, (n * (n + 1)) // 2, '--------------')
      n = (n * (n + 1)) // 2 if self.interaction_itself else (n * (n - 1)) // 2
      # x.set_shape((shape[0], n))
      # x -= 1e10
      x = tf.reshape(x, (-1, n))
    return x
//  [python]
def call(self, x, sequence_length=None, axis=1):
    q, k, v = x, x, x
      
    mask = None if sequence_length is None else (1. - tf.sequence_mask(sequence_length, dtype=x.dtype))
    if mask is not None:
      mask = tf.cast(mask, x.dtype)
      mask = mask[:, tf.newaxis, tf.newaxis, :]

    batch_size = tf.shape(q)[0]
    
    q = self.wq(q)  # (batch_size, seq_len, d_model)
    k = self.wk(k)  # (batch_size, seq_len, d_model)
    v = self.wv(v)  # (batch_size, seq_len, d_model)
    
    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)
    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)
    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)

    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
    scaled_attention, attention_weights = melt.scaled_dot_product_attention(
        q, k, v, mask)

    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)

    concat_attention = tf.reshape(scaled_attention, 
                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)

    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)
    
    output = self.pooling(output, sequence_length)
    if self.return_att:
      return output, attention_weights
    else:
      return output
//  [python]
def call(self, x, value=None, combiner=None):
    if x is not None:
      if isinstance(x, tf.Tensor):
        if self.need_mod:
          mask = t.cast(K.not_equal(x, 0), x.dtype)
          x = x % self.input_dim
          x = (x + 1) * mask
        return tf.nn.embedding_lookup(self.embeddings, x) * tf.nn.embedding_lookup(self.embeddings2, tf.zeros_like(x))
      else:
        if isinstance(x, (list, tuple)):
          x, value = x[0], x[1]
        if self.need_mod:
          x = tf.sparse.SparseTensor(x.indices, x.values % self.input_dim, x.dense_shape)
        return tf.nn.embedding_lookup_sparse(self.embeddings, x, value, combiner=self.mode)
    else:
      return self.embeddings
//  [python]
def call(self, x, value=None, combiner=None):
    if x is not None:
      if isinstance(x, tf.Tensor):
        if self.need_mod:
          mask = tf.cast(K.not_equal(x, 0), x.dtype)
          x = x % self.input_dim
          x = (x + 1) * mask
        return tf.nn.embedding_lookup(self.embeddings, x) * tf.nn.embedding_lookup(self.embeddings2, tf.zeros_like(x))
      else:
        if isinstance(x, (list, tuple)):
          x, value = x[0], x[1]
        if self.need_mod:
          x = tf.sparse.SparseTensor(x.indices, x.values % self.input_dim, x.dense_shape)
        return tf.nn.embedding_lookup_sparse(self.embeddings, x, value, combiner=self.mode)
    else:
      return self.embeddings
//  [python]
def interpolate(im,x,y):
            
    im = tf.pad(im,[[0,0],[1,1],[1,1],[0,0]],"REFLECT")
                
    num_batch = tf.shape(im)[0]
    height = tf.shape(im)[1]
    width = tf.shape(im)[2]
    channels = tf.shape(im)[3]

    out_height = tf.shape(x)[1]
    out_width = tf.shape(x)[2]
                
    x = tf.reshape(x,[-1])
    y = tf.reshape(y,[-1])
                
    x = tf.to_float(x)+1
    y = tf.to_float(y)+1
                
    max_x = tf.to_int32(width - 1)
    max_y = tf.to_int32(height - 1)
                
    x0 = tf.to_int32(tf.floor(x))
    x1 = x0 + 1
    y0 = tf.to_int32(tf.floor(y))
    y1 = y0 + 1
                
    x0 = tf.clip_by_value(x0, 0, max_x)
    x1 = tf.clip_by_value(x1, 0, max_x)
    y0 = tf.clip_by_value(y0, 0, max_y)
    y1 = tf.clip_by_value(y1, 0, max_y)
                
    base = repeat(tf.range(num_batch)*width*height, (out_height*out_width))

    base_y0 = base + y0*width
    base_y1 = base + y1*width

    idx_a = base_y0 + x0
    idx_b = base_y1 + x0
    idx_c = base_y0 + x1
    idx_d = base_y1 + x1
                
    # use indices to lookup pixels in the flat image and restore
    # channels dim
    im_flat = tf.reshape(im, tf.stack([-1, channels]))
    im_flat = tf.to_float(im_flat)
                
    Ia = tf.gather(im_flat, idx_a)
    Ib = tf.gather(im_flat, idx_b)
    Ic = tf.gather(im_flat, idx_c)
    Id = tf.gather(im_flat, idx_d)
                
    # and finally calculate interpolated values
    x1_f = tf.to_float(x1)
    y1_f = tf.to_float(y1)
                
    dx = x1_f - x
    dy = y1_f - y
                
    wa = tf.expand_dims((dx * dy), 1)
    wb = tf.expand_dims((dx * (1-dy)), 1)
    wc = tf.expand_dims(((1-dx) * dy), 1)
    wd = tf.expand_dims(((1-dx) * (1-dy)), 1)
                
    output = tf.add_n([wa*Ia, wb*Ib, wc*Ic, wd*Id])
    output = tf.reshape(output, tf.stack([-1,out_height,out_width,channels]))
    return output
//  [python]
def interpolate(im,x,y):
            
    im = tf.pad(im,[[0,0],[1,1],[1,1],[0,0]],"REFLECT")
                
    num_batch = tf.shape(im)[0]
    height = tf.shape(im)[1]
    width = tf.shape(im)[2]
    channels = tf.shape(im)[3]

    out_height = tf.shape(x)[1]
    out_width = tf.shape(x)[2]
                
    x = tf.reshape(x,[-1])
    y = tf.reshape(y,[-1])
                
    x = tf.cast(x, 'float32')+1
    y = tf.cast(y, 'float32')+1
                
    max_x = tf.cast(width - 1, 'int32')
    max_y = tf.cast(height - 1, 'int32')
                
    x0 = tf.cast(tf.floor(x), 'int32')
    x1 = x0 + 1
    y0 = tf.cast(tf.floor(y), 'int32')
    y1 = y0 + 1
                
    x0 = tf.clip_by_value(x0, 0, max_x)
    x1 = tf.clip_by_value(x1, 0, max_x)
    y0 = tf.clip_by_value(y0, 0, max_y)
    y1 = tf.clip_by_value(y1, 0, max_y)
                
    base = repeat(tf.range(num_batch)*width*height, (out_height*out_width))

    base_y0 = base + y0*width
    base_y1 = base + y1*width

    idx_a = base_y0 + x0
    idx_b = base_y1 + x0
    idx_c = base_y0 + x1
    idx_d = base_y1 + x1
                
    # use indices to lookup pixels in the flat image and restore
    # channels dim
    im_flat = tf.reshape(im, tf.stack([-1, channels]))
    im_flat = tf.cast(im_flat, 'float32')
                
    Ia = tf.gather(im_flat, idx_a)
    Ib = tf.gather(im_flat, idx_b)
    Ic = tf.gather(im_flat, idx_c)
    Id = tf.gather(im_flat, idx_d)
                
    # and finally calculate interpolated values
    x1_f = tf.cast(x1, 'float32')
    y1_f = tf.cast(y1, 'float32')
                
    dx = x1_f - x
    dy = y1_f - y
                
    wa = tf.expand_dims((dx * dy), 1)
    wb = tf.expand_dims((dx * (1-dy)), 1)
    wc = tf.expand_dims(((1-dx) * dy), 1)
    wd = tf.expand_dims(((1-dx) * (1-dy)), 1)
                
    output = tf.add_n([wa*Ia, wb*Ib, wc*Ic, wd*Id])
    output = tf.reshape(output, tf.stack([-1,out_height,out_width,channels]))
    return output
//  [python]
def call(self, x, value=None, combiner=None, pooling=False):
    mask = t.cast(K.not_equal(x, 0), x.dtype)

    if self.need_mod:
      x = x % self.input_dim

    if self.input_dim == self.num_buckets:
      return self.embeddings(x)
    
    # TODO why tf.cast to int32 will turn much slower *2 times..
    x_rem = (x % self.num_buckets + 1) * mask
    # NOTICE do not use cast like tf.cast(x/self.num_buckets, tf.int32)
    x_quo = (x // self.num_buckets + 1) * mask
    embs = self.embeddings(x_rem)
    if self.num_quotients > 1:
      embs2 = self.embeddings2(x_quo)

    combiner = combiner or self.combiner
    if combiner == 'mul':
      embs = embs * embs2
    elif combiner == 'sum':
      # -> [batch_size, len, out_dim]
      embs = embs + embs2
    elif combiner == 'concat':
      embs = t.concat([embs, embs2], -1)
    else:
      raise ValueError(combiner)
    if pooling:
      if value is not None:
        embs *= t.expand_dims(value, -1)
        len_ = None if self.mode == 'sum' else melt.length(x)
      else:
        len_ = melt.length(x) 
      embs = self.pooling(embs, len_)
    return embs
//  [python]
def call(self, x, value=None, combiner=None, pooling=False):
    mask = tf.cast(K.not_equal(x, 0), x.dtype)

    if self.need_mod:
      x = x % self.input_dim

    if self.input_dim == self.num_buckets:
      return self.embeddings(x)
    
    # TODO why tf.cast to int32 will turn much slower *2 times..
    x_rem = (x % self.num_buckets + 1) * mask
    # NOTICE do not use cast like tf.cast(x/self.num_buckets, tf.int32)
    x_quo = (x // self.num_buckets + 1) * mask
    embs = self.embeddings(x_rem)
    if self.num_quotients > 1:
      embs2 = self.embeddings2(x_quo)

    combiner = combiner or self.combiner
    if combiner == 'mul':
      embs = embs * embs2
    elif combiner == 'sum':
      # -> [batch_size, len, out_dim]
      embs = embs + embs2
    elif combiner == 'concat':
      embs = tf.concat([embs, embs2], -1)
    else:
      raise ValueError(combiner)
    if pooling:
      if value is not None:
        embs *= tf.expand_dims(value, -1)
        len_ = None if self.mode == 'sum' else melt.length(x)
      else:
        len_ = melt.length(x) 
      embs = self.pooling(embs, len_)
    return embs
//  [python]
def call(self, query, keys, sequence_length=None, axis=1):
    q, k, v = query, keys, keys
      
    mask = None if sequence_length is None else (1. - tf.sequence_mask(sequence_length, dtype=query.dtype))
    if mask is not None:
      mask = t.cast(mask, query.dtype)
      mask = mask[:, tf.newaxis, tf.newaxis, :]

    batch_size = tf.shape(q)[0]
    
    q = self.wq(q)  # (batch_size, seq_len, d_model)
    k = self.wk(k)  # (batch_size, seq_len, d_model)
    v = self.wv(v)  # (batch_size, seq_len, d_model)
    
    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)
    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)
    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)

    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
    scaled_attention, attention_weights = melt.scaled_dot_product_attention(
        q, k, v, mask)

    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)

    concat_attention = tf.reshape(scaled_attention, 
                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)

    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)
    
    output = self.pooling(output, sequence_length)
    if self.return_att:
      return output, attention_weights
    else:
      return output
//  [python]
def call(self, query, keys, sequence_length=None, axis=1):
    q, k, v = query, keys, keys
      
    mask = None if sequence_length is None else (1. - tf.sequence_mask(sequence_length, dtype=query.dtype))
    if mask is not None:
      mask = tf.cast(mask, query.dtype)
      mask = mask[:, tf.newaxis, tf.newaxis, :]

    batch_size = tf.shape(q)[0]
    
    q = self.wq(q)  # (batch_size, seq_len, d_model)
    k = self.wk(k)  # (batch_size, seq_len, d_model)
    v = self.wv(v)  # (batch_size, seq_len, d_model)
    
    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)
    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)
    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)

    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
    scaled_attention, attention_weights = melt.scaled_dot_product_attention(
        q, k, v, mask)

    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)

    concat_attention = tf.reshape(scaled_attention, 
                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)

    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)
    
    output = self.pooling(output, sequence_length)
    if self.return_att:
      return output, attention_weights
    else:
      return output
//  [python]
def call(self, keys, sequence_length=None, axis=1):
    q, k, v = keys, keys, keys
      
    seq_len = melt.get_shape(q, 1)
    mask = None if sequence_length is None else (1. - tf.sequence_mask(sequence_length, maxlen=seq_len, dtype=q.dtype))
    if mask is not None:
      mask = t.cast(mask, q.dtype)
      mask = mask[:, tf.newaxis, tf.newaxis, :]

    batch_size = tf.shape(q)[0]
    
    q = self.wq(q)  # (batch_size, seq_len, d_model)
    k = self.wk(k)  # (batch_size, seq_len, d_model)
    v = self.wv(v)  # (batch_size, seq_len, d_model)
    
    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)
    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)
    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)

    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
    scaled_attention, attention_weights = melt.scaled_dot_product_attention(
        q, k, v, mask)

    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)

    concat_attention = tf.reshape(scaled_attention, 
                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)

    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)
    
    if self.return_att:
      return output, attention_weights
    else:
      return output
//  [python]
def call(self, keys, sequence_length=None, axis=1):
    q, k, v = keys, keys, keys
      
    seq_len = melt.get_shape(q, 1)
    mask = None if sequence_length is None else (1. - tf.sequence_mask(sequence_length, maxlen=seq_len, dtype=q.dtype))
    if mask is not None:
      mask = tf.cast(mask, q.dtype)
      mask = mask[:, tf.newaxis, tf.newaxis, :]

    batch_size = tf.shape(q)[0]
    
    q = self.wq(q)  # (batch_size, seq_len, d_model)
    k = self.wk(k)  # (batch_size, seq_len, d_model)
    v = self.wv(v)  # (batch_size, seq_len, d_model)
    
    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)
    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)
    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)

    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
    scaled_attention, attention_weights = melt.scaled_dot_product_attention(
        q, k, v, mask)

    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)

    concat_attention = tf.reshape(scaled_attention, 
                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)

    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)
    
    if self.return_att:
      return output, attention_weights
    else:
      return output
//  [python]
def call(self, keys, keys_len, query, context=None):
    '''
    queries:     [B, H]
    keys:        [B, T, H]
    keys_length: [B]
    '''
    assert query != None
    dtype = keys.dtype
    if isinstance(keys, list or tuple):
      query, keys, keys_len = keys
      keys = t.cast(keys, dtype)
      query = t.cast(query, dtype)

    T = melt.get_shape(keys, 1)
    HIDDEN = melt.get_shape(keys, 2)
    query = tf.tile(query, [1, T])
    query = tf.reshape(query, [-1, T, HIDDEN])
    if context is not None:
      query = tf.concat([query, context], axis=-1)
      query = self.dense(query)
    keys = t.cast(keys, dtype)
    query = t.cast(query, dtype)
    din_all = tf.concat([query, keys, query - keys, query * keys], axis=-1)
    outputs = self.mlp(din_all) # [B, T, 1]
    outputs = tf.reshape(outputs, [-1, 1, T]) # [B, 1, T]
    # Mask
    key_masks = tf.sequence_mask(keys_len, T)   # [B, T]
    key_masks = tf.expand_dims(key_masks, 1) # [B, 1, T]
    if self.weight_normalization:
      # paddings = tf.ones_like(outputs) * (-2 ** 32 + 1)
      paddings = tf.ones_like(outputs) * (-2 ** 15 + 1)
    else:
      paddings = tf.zeros_like(outputs)

    outputs = tf.where(key_masks, outputs, paddings)  # [B, 1, T]
    outputs = t.cast(outputs, dtype)
    # Scale
    outputs = outputs / (t.cast(HIDDEN, dtype) ** 0.5)

    # Activation
    if self.weight_normalization:
      outputs = tf.nn.softmax(outputs)  # [B, 1, T]

    # Weighted sum
    keys = t.cast(keys, dtype)
    outputs = tf.matmul(outputs, keys)  # [B, 1, H]
    outputs = tf.squeeze(outputs, 1)

    return outputs
//  [python]
def call(self, keys, keys_len, query, context=None):
    '''
    queries:     [B, H]
    keys:        [B, T, H]
    keys_length: [B]
    '''
    assert query != None
    dtype = keys.dtype
    if isinstance(keys, list or tuple):
      query, keys, keys_len = keys
      keys = tf.cast(keys, dtype)
      query = tf.cast(query, dtype)

    T = melt.get_shape(keys, 1)
    HIDDEN = melt.get_shape(keys, 2)
    query = tf.tile(query, [1, T])
    query = tf.reshape(query, [-1, T, HIDDEN])
    if context is not None:
      query = tf.concat([query, context], axis=-1)
      query = self.dense(query)
    keys = tf.cast(keys, dtype)
    query = tf.cast(query, dtype)
    din_all = tf.concat([query, keys, query - keys, query * keys], axis=-1)
    outputs = self.mlp(din_all) # [B, T, 1]
    outputs = tf.reshape(outputs, [-1, 1, T]) # [B, 1, T]
    # Mask
    key_masks = tf.sequence_mask(keys_len, T)   # [B, T]
    key_masks = tf.expand_dims(key_masks, 1) # [B, 1, T]
    if self.weight_normalization:
      # paddings = tf.ones_like(outputs) * (-2 ** 32 + 1)
      paddings = tf.ones_like(outputs) * (-2 ** 15 + 1)
    else:
      paddings = tf.zeros_like(outputs)

    outputs = tf.where(key_masks, outputs, paddings)  # [B, 1, T]
    outputs = tf.cast(outputs, dtype)
    # Scale
    outputs = outputs / (tf.cast(HIDDEN, dtype) ** 0.5)

    # Activation
    if self.weight_normalization:
      outputs = tf.nn.softmax(outputs)  # [B, 1, T]

    # Weighted sum
    keys = tf.cast(keys, dtype)
    outputs = tf.matmul(outputs, keys)  # [B, 1, H]
    outputs = tf.squeeze(outputs, 1)

    return outputs
//  [python]
def test_fit_with_all_types(self):
        schema = Schema.new_builder() \
            .column("x1", DataTypes.STRING()) \
            .column("x2", DataTypes.INT()) \
            .column("x3", DataTypes.BIGINT()) \
            .column("x4", DataTypes.FLOAT()) \
            .column("x5", DataTypes.DOUBLE()) \
            .column_by_expression("y", expr.call_sql("2 * x2")) \
            .build()
        input_table = self.t_env.from_descriptor(TableDescriptor
                                                 .for_connector("datagen")
                                                 .schema(schema)
                                                 .option('number-of-rows',
                                                         '100')
                                                 .option('fields.x2.min', '0')
                                                 .option('fields.x2.max', '100')
                                                 .build())

        def select_inputs(inputs):
            import tensorflow as tf
            inputs = tf.keras.backend.print_tensor(inputs)
            return tf.cast(inputs[1], tf.float32)

        input1 = tf.keras.Input(shape=(1,), dtype=tf.string, name="x1")
        input2 = tf.keras.Input(shape=(1,), dtype=tf.int32, name="x2")
        input3 = tf.keras.Input(shape=(1,), dtype=tf.int64, name="x3")
        input4 = tf.keras.Input(shape=(1,), dtype=tf.float32, name="x4")
        input5 = tf.keras.Input(shape=(1,), dtype=tf.float64, name="x5")
        output = tf.keras.layers.Lambda(select_inputs)(
            [input1, input2, input3, input4, input5])
        output = tf.keras.layers.Dense(units=1)(output)

        model = tf.keras.Model(inputs=[input1, input2, input3, input4, input5],
                               outputs=output)
        loss = tf.keras.losses.MeanSquaredError()

        estimator = TFEstimator(self.statement_set, model, loss, 'adam', 1,
                                ["x1", "x2", "x3", "x4", "x5"],
                                "y", batch_size=1)
        model = estimator.fit(input_table)
        model_path = self._get_model_path()
        model.save(model_path)
        self.statement_set.execute().wait()

        table = model.transform(input_table.drop_columns("y").fetch(10))[0]
        table.execute().print()
//  [python]
def call(self, x, value=None, combiner=None, pooling=False):
    if x is not None:
      if isinstance(x, tf.Tensor):
        mask = tf.cast(K.not_equal(x, 0), x.dtype)

        if self.need_mod:
          x = x % self.input_dim
          # x = tf.math.floormod(x, self.input_dim)

        if self.input_dim == self.num_buckets:
          return tf.nn.embedding_lookup(self.embeddings, x)
        
        # TODO why tf.cast to int32 will turn much slower *2 times..
        x_rem = (x % self.num_buckets + 1) * mask
        # x_rem = (tf.math.floormod(x, self.num_buckets) + 1) * mask
        # NOTICE do not use cast like tf.cast(x/self.num_buckets, tf.int32)
        x_quo = (x // self.num_buckets + 1) * mask
        embs = tf.nn.embedding_lookup(self.embeddings, x_rem)
        if self.num_quotients > 1:
          embs2 = tf.nn.embedding_lookup(self.embeddings2, x_quo)
        # if pooling:
        #   if value is not None:
        #     embs *= tf.expand_dims(value, -1)
        #     len_ = None if self.mode == 'sum' else melt.length(x)
        #   else:
        #     len_ = melt.length(x) 
        #   embs = self.pooling(embs, len_)
        #   embs2 = self.pooling(embs2, len_)
      else:
        if isinstance(x, (list, tuple)):
          x, value = x[0], x[1]
        if self.need_mod:
          x = tf.sparse.SparseTensor(x.indices, x.values % self.input_dim, x.dense_shape)
        x_rem = x.values % self.num_buckets
        x_quo = x.values // self.num_buckets
        x_rem = tf.sparse.SparseTensor(x.indices, x_rem, x.dense_shape)
        x_quo = tf.sparse.SparseTensor(x.indices, x_quo, x.dense_shape)
        embs = tf.nn.embedding_lookup_sparse(self.embeddings, x_rem, value, combiner=self.mode)
        embs2 = tf.nn.embedding_lookup_sparse(self.embeddings2, x_quo, value, combiner=self.mode)

      combiner = combiner or self.combiner
      if combiner == 'mul':
        embs = embs * embs2
      elif combiner == 'sum':
        # -> [batch_size, len, out_dim]
        embs = embs + embs2
      elif combiner == 'concat':
        embs = tf.concat([embs, embs2], -1)
      else:
        raise ValueError(combiner)
      if pooling:
        if value is not None:
          embs *= tf.expand_dims(value, -1)
          len_ = None if self.mode == 'sum' else melt.length(x)
        else:
          len_ = melt.length(x) 
        embs = self.pooling(embs, len_)
      return embs
    else:
      return self.embeddings
//  [python]
def highlight_layer(inputs,
                    labels,
                    mask,
                    epsilon=1e-12,
                    reuse=None,
                    name='highlight_layer'):
    with tf.variable_scope(name, reuse=reuse):
        logits = conv1d(inputs,
                        dim=1,
                        use_bias=True,
                        padding='VALID',
                        reuse=reuse,
                        name='dense')
        logits = tf.squeeze(logits, axis=-1)  # (batch_size, seq_length)
        logits = mask_logits(logits, mask=mask)

        # prepare labels and weights
        labels = tf.to_float(labels)
        weights = tf.where(tf.equal(labels, 0.0),
                           x=labels + 1.0,
                           y=labels * 2.0)
        
        labels = tf.clip_by_value(labels, 1e-7, 1.0)

        # binary cross entropy with sigmoid activation
        loss_per_location = tf.nn.sigmoid_cross_entropy_with_logits(
            labels=labels, logits=logits)
        loss_per_location = loss_per_location * weights
        mask = tf.to_float(mask)
        loss = tf.reduce_sum(
            loss_per_location * mask) / (tf.reduce_sum(mask) + epsilon)

        # compute scores
        scores = tf.sigmoid(logits)
        return loss, scores
//  [python]
def highlight_layer(inputs,
                    labels,
                    mask,
                    epsilon=1e-12,
                    reuse=None,
                    name='highlight_layer'):
    with tf.variable_scope(name, reuse=reuse):
        logits = conv1d(inputs,
                        dim=1,
                        use_bias=True,
                        padding='VALID',
                        reuse=reuse,
                        name='dense')
        logits = tf.squeeze(logits, axis=-1)  # (batch_size, seq_length)
        logits = mask_logits(logits, mask=mask)

        # prepare labels and weights
        labels = tf.cast(labels, dtype=logits.dtype)
        weights = tf.where(tf.equal(labels, 0.0),
                           x=labels + 1.0,
                           y=labels * 2.0)
        
        labels = tf.clip_by_value(labels, 1e-7, 1.0)

        # binary cross entropy with sigmoid activation
        loss_per_location = tf.nn.sigmoid_cross_entropy_with_logits(
            labels=labels, logits=logits)
        loss_per_location = loss_per_location * weights
        mask = tf.cast(mask, dtype=logits.dtype)
        loss = tf.reduce_sum(
            loss_per_location * mask) / (tf.reduce_sum(mask) + epsilon)

        # compute scores
        scores = tf.sigmoid(logits)
        return loss, scores
//  [python]
def smooth_l1_loss(labels, predictions, scope=tf.GraphKeys.LOSSES):
    """smooth_l1_loss

    Args:
        labels ([type]): [description]
        predictions ([type]): [description]
        scope ([type], optional): [description]. Defaults to tf.GraphKeys.LOSSES.

    Returns:
        [type]: [description]
    """
    with tf.variable_scope(scope):
        diff = tf.abs(labels - predictions)
        less_than_one = tf.to_float(tf.less(diff, 0.5))  # Bool to float32
        smooth_l1_loss = (less_than_one * diff ** 2) + \
            (1.0 - less_than_one) * (diff - 0.25)  # 同上图公式
        return tf.reduce_mean(smooth_l1_loss)
//  [python]
def smooth_l1_loss(labels, predictions, scope=tf.GraphKeys.LOSSES):
    """smooth_l1_loss

    Args:
        labels ([type]): [description]
        predictions ([type]): [description]
        scope ([type], optional): [description]. Defaults to tf.GraphKeys.LOSSES.

    Returns:
        [type]: [description]
    """
    with tf.variable_scope(scope):
        diff = tf.abs(labels - predictions)
        less_than_one = tf.cast(tf.less(diff, 0.5),
                                tf.float32)  # Bool to float32
        smooth_l1_loss = (less_than_one * diff ** 2) + \
            (1.0 - less_than_one) * (diff - 0.25)  # 同上图公式
        return tf.reduce_mean(smooth_l1_loss)
//  [python]
def pairwise_distance(feature: TensorLike, squared: bool = False):
    """Computes the pairwise distance matrix with numerical stability.

    output[i, j] = || feature[i, :] - feature[j, :] ||_2

    Args:
      feature: 2-D Tensor of size `[number of data, feature dimension]`.
      squared: Boolean, whether or not to square the pairwise distances.

    Returns:
      pairwise_distances: 2-D Tensor of size `[number of data, number of data]`.
    """
    pairwise_distances_squared = tf.math.add(
        tf.math.reduce_sum(tf.math.square(feature), axis=[1], keepdims=True),
        tf.math.reduce_sum(
            tf.math.square(tf.transpose(feature)), axis=[0], keepdims=True
        ),
    ) - 2.0 * tf.matmul(feature, tf.transpose(feature))

    # Deal with numerical inaccuracies. Set small negatives to zero.
    pairwise_distances_squared = tf.math.maximum(pairwise_distances_squared, 0.0)
    # Get the mask where the zero distances are at.
    error_mask = tf.math.less_equal(pairwise_distances_squared, 0.0)

    # Optionally take the sqrt.
    if squared:
        pairwise_distances = pairwise_distances_squared
    else:
        pairwise_distances = tf.math.sqrt(
            pairwise_distances_squared
            + tf.cast(error_mask, dtype=tf.dtypes.float32) * 1e-16
        )

    # Undo conditionally adding 1e-16.
    pairwise_distances = tf.math.multiply(
        pairwise_distances,
        tf.cast(tf.math.logical_not(error_mask), dtype=tf.dtypes.float32),
    )

    num_data = tf.shape(feature)[0]
    # Explicitly set diagonals to zero.
    mask_offdiagonals = tf.ones_like(pairwise_distances) - tf.linalg.diag(
        tf.ones([num_data])
    )
    pairwise_distances = tf.math.multiply(pairwise_distances, mask_offdiagonals)
    return pairwise_distances
//  [python]
def multi_modal_sa(vfeats, qfeats, vmask, qmask, qlength, reuse=None, name='multi_modal_self_attention'):
    with tf.variable_scope(name, reuse=reuse):
        b, lv, dv = get_shape_list(vfeats)
        b, lq, dq = get_shape_list(qfeats)

        vfeats = tf.expand_dims(vfeats, 1)
        qfeats = tf.expand_dims(qfeats, 2)

        vfeats = tf.tile(vfeats, [1, lq, 1, 1])
        qfeats = tf.tile(qfeats, [1, 1, lv, 1])

        outputs = tf.reshape(tf.concat([vfeats, qfeats], -1), [b*lq, lv, dv+dq])

        vmask = tf.tile(tf.expand_dims(vmask, 1), [1, lq, 1])

        outputs = conv1d(outputs, dim=512, use_bias=True, reuse=False, name='mm_dense')
        outputs = multihead_attention_block(outputs, 512, num_heads=1, mask=vmask, use_bias=True, drop_rate=0.0, reuse=None, name='multihead_attention_block')

        outputs = tf.reshape(outputs, [b, lq, lv, -1])

        qmask = tf.expand_dims(tf.tile(tf.expand_dims(qmask, -1), [1, 1, lv]), -1)

        outputs = qmask * outputs #[b, lq, lv, d]

        # outputs = tf.reduce_sum(outputs, 1) #[b, lv, d]
        outputs = tf.reduce_max(outputs, 1)
        # qlength = tf.tile(tf.reshape(qlength, [b, 1, 1]), [1, lv, 1])
        # outputs = tf.divide(outputs, tf.cast(qlength, tf.float32))
        outputs = tf.reshape(outputs, [b, lv, 512])

        outputs = conv1d(outputs, dim=dv, use_bias=True, reuse=False, name='mmsa_dense')
        return outputs
//  [python]
def multi_modal_sa(vfeats, qfeats, vmask, qmask, qlength, reuse=None, name='multi_modal_self_attention'):
    with tf.variable_scope(name, reuse=reuse):
        b, lv, dv = get_shape_list(vfeats)
        b, lq, dq = get_shape_list(qfeats)

        vfeats = tf.expand_dims(vfeats, 1)
        qfeats = tf.expand_dims(qfeats, 2)

        vfeats = tf.tile(vfeats, [1, lq, 1, 1])
        qfeats = tf.tile(qfeats, [1, 1, lv, 1])

        outputs = tf.reshape(tf.concat([vfeats, qfeats], -1), [b*lq, lv, dv+dq])

        vmask = tf.tile(tf.expand_dims(vmask, 1), [1, lq, 1])

        outputs = conv1d(outputs, dim=512, use_bias=True, reuse=False, name='mm_dense')
        outputs = multihead_attention_block(outputs, 512, num_heads=1, mask=vmask, use_bias=True, drop_rate=0.0, reuse=None, name='multihead_attention_block')

        outputs = tf.reshape(outputs, [b, lq, lv, -1])

        qmask = tf.cast(tf.expand_dims(tf.tile(tf.expand_dims(qmask, -1), [1, 1, lv]), -1), tf.float32)

        outputs = qmask * outputs #[b, lq, lv, d]

        # outputs = tf.reduce_sum(outputs, 1) #[b, lv, d]
        outputs = tf.reduce_max(outputs, 1)
        # qlength = tf.tile(tf.reshape(qlength, [b, 1, 1]), [1, lv, 1])
        # outputs = tf.divide(outputs, tf.cast(qlength, tf.float32))
        outputs = tf.reshape(outputs, [b, lv, 512])

        outputs = conv1d(outputs, dim=dv, use_bias=True, reuse=False, name='mmsa_dense')
        return outputs
//  [python]
def generate_proposal_boxes(vfeats, length, configs):

    batch_size = tf.shape(vfeats)[0]

    if configs.mode == "train":
        k = 100
    elif configs.mode == "test":
        k = 100

    proposal_box0 = tf.random_uniform([k, 1], maxval=1.0)
    # proposal_box1 = tf.random_uniform([k, 1], minval=0.0, maxval=0.5)
    proposal_box1 = tf.random_uniform([k, 1], minval=0.0, maxval=0.25)
    proposal_box = tf.Variable(tf.concat([proposal_box0, proposal_box1], 1), trainable=True, name='proposal_box')

    l = tf.expand_dims(tf.to_float(length), -1)
    a = proposal_box[:, 0]
    b = proposal_box[:, 1]

    dx = tf.to_int32(tf.floor(tf.matmul(l, tf.expand_dims((a - b), -1), transpose_b=True)))
    dy = tf.to_int32(tf.ceil(tf.matmul(l, tf.expand_dims((a + b), -1), transpose_b=True)))

    ones = tf.ones([k, 1], dtype=tf.float32)
    zeros = tf.zeros([k, 1], dtype=tf.float32)

    starts = tf.expand_dims(a - b, -1)
    ends = tf.expand_dims(a + b, -1)
    boxes = tf.nn.relu(tf.concat([zeros, starts, ones, ends], 1))  # [k*4]

    return proposal_box, dx, dy, boxes
//  [python]
def generate_proposal_boxes(vfeats, length, configs):

    batch_size = tf.shape(vfeats)[0]

    if configs.mode == "train":
        k = 100
    elif configs.mode == "test":
        k = 100

    proposal_box0 = tf.random_uniform([k, 1], maxval=1.0)
    # proposal_box1 = tf.random_uniform([k, 1], minval=0.0, maxval=0.5)
    proposal_box1 = tf.random_uniform([k, 1], minval=0.0, maxval=0.25)
    proposal_box = tf.Variable(tf.concat([proposal_box0, proposal_box1], 1), trainable=True, name='proposal_box')

    l = tf.expand_dims(tf.cast(length, tf.float32), -1)
    a = proposal_box[:, 0]
    b = proposal_box[:, 1]

    dx = tf.cast(tf.floor(tf.matmul(l, tf.expand_dims((a - b), -1), transpose_b=True)), tf.int32)
    dy = tf.cast(tf.ceil(tf.matmul(l, tf.expand_dims((a + b), -1), transpose_b=True)), tf.int32)

    ones = tf.ones([k, 1], dtype=tf.float32)
    zeros = tf.zeros([k, 1], dtype=tf.float32)

    starts = tf.expand_dims(a - b, -1)
    ends = tf.expand_dims(a + b, -1)
    boxes = tf.nn.relu(tf.concat([zeros, starts, ones, ends], 1))  # [k*4]

    return proposal_box, dx, dy, boxes