jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def _weights(config, masks, curriculum_step=None):
    """ Returns dRMSD weights that mask meaningless (missing or longer than 
        sequence residues) pairwise distances and incorporate the state of 
        the curriculum to differentially weigh pairwise distances based on 
        their proximity. """

    if config['atoms'] == 'c_alpha':
        if config['mode'] != 'loss':
            # no loss-based curriculum, create fixed weighting matrix that weighs all distances equally. 
            # minus one factor is there because we ignore self-distances.
            flat_curriculum_weights = np.ones(config['num_steps'] - config['num_edge_residues'] - 1, dtype='float32')

        elif config['mode'] == 'loss' and curriculum_step is not None:
            # create appropriate weights based on curriculum parameters and current step.
            flat_curriculum_weights = curriculum_weights(base=curriculum_step, 
                                                         slope=config['slope'], 
                                                         max_seq_length=config['num_steps'] - config['num_edge_residues'])
        else:
            raise RuntimeError('Curriculum step tensor not supplied.')

        # weighting matrix for entire batch that accounts for curriculum weighting.
        unnormalized_weights = weighting_matrix(flat_curriculum_weights, name='unnormalized_weights')
                               # [NUM_STEPS - NUM_EDGE_RESIDUES, NUM_STEPS - NUM_EDGE_RESIDUES]

        # create final weights by multiplying with masks and normalizing.
        mask_length = tf.shape(masks)[0]
        unnormalized_masked_weights = masks * unnormalized_weights[:mask_length, :mask_length, tf.newaxis]
        masked_weights = tf.div(unnormalized_masked_weights, 
                                tf.reduce_sum(unnormalized_masked_weights, axis=[0, 1]), 
                                name='weights')

        return masked_weights, flat_curriculum_weights

    else:
        raise NotImplementedError('Model does not currently support anything other than C alpha atoms for the loss function.')
//  [python]
def _weights(config, masks, curriculum_step=None):
    """ Returns dRMSD weights that mask meaningless (missing or longer than 
        sequence residues) pairwise distances and incorporate the state of 
        the curriculum to differentially weigh pairwise distances based on 
        their proximity. """

    if config['atoms'] == 'c_alpha':
        if config['mode'] != 'loss':
            # no loss-based curriculum, create fixed weighting matrix that weighs all distances equally. 
            # minus one factor is there because we ignore self-distances.
            flat_curriculum_weights = np.ones(config['num_steps'] - config['num_edge_residues'] - 1, dtype='float32')

        elif config['mode'] == 'loss' and curriculum_step is not None:
            # create appropriate weights based on curriculum parameters and current step.
            flat_curriculum_weights = curriculum_weights(base=curriculum_step, 
                                                         slope=config['slope'], 
                                                         max_seq_length=config['num_steps'] - config['num_edge_residues'])
        else:
            raise RuntimeError('Curriculum step tensor not supplied.')

        # weighting matrix for entire batch that accounts for curriculum weighting.
        unnormalized_weights = weighting_matrix(flat_curriculum_weights, name='unnormalized_weights')
                               # [NUM_STEPS - NUM_EDGE_RESIDUES, NUM_STEPS - NUM_EDGE_RESIDUES]

        # create final weights by multiplying with masks and normalizing.
        mask_length = tf.shape(masks)[0]
        unnormalized_masked_weights = masks * unnormalized_weights[:mask_length, :mask_length, tf.newaxis]
        masked_weights = tf.divide(unnormalized_masked_weights, 
                                tf.reduce_sum(unnormalized_masked_weights, axis=[0, 1]), 
                                name='weights')

        return masked_weights, flat_curriculum_weights

    else:
        raise NotImplementedError('Model does not currently support anything other than C alpha atoms for the loss function.')
//  [python]
def __call__(self, brboxes, expand_w=20):
		paddings = tf.constant([[0, 0],[expand_w, expand_w], [expand_w, expand_w], [0, 0]])
		features_pad = tf.pad(self.features, paddings, "CONSTANT")
		features_pad = tf.expand_dims(features_pad, axis=1)
		# features_pad shape: [b, 1, h, w, c]
		nums = features_pad.shape[0]
		channels = features_pad.shape[-1]

		btextImgFeatures = []
		ws = []

		for b, rboxes in enumerate(brboxes):
			outBoxes, cropBoxes, angles = rboxes
			# outBoxes = tf.cast(tf.ceil(tf.divide(outBoxes, self.features_stride)), tf.int32)  # float div
			# cropBoxes = tf.cast(tf.ceil(tf.divide(cropBoxes, self.features_stride)), tf.int32) # float div

			outBoxes = tf.div(outBoxes, self.features_stride)  # int div
			cropBoxes = tf.div(cropBoxes, self.features_stride) # int div

			outBoxes_xy = outBoxes[:, :2]
			outBoxes_xy =  tf.add(outBoxes_xy, expand_w)
			outBoxes = tf.concat([outBoxes_xy, outBoxes[:, 2:]], axis=1)

			# len_crop = outBoxes.shape[0]  # error tf.stack cannot convert an unknown Dimension to a tensor: ?
			len_crop = tf.shape(outBoxes)[0]
			ifeatures_pad = features_pad[b]
			# ifeatures_tile = tf.tile(ifeatures_pad, tf.stack([len_crop, 1, 1, 1]))
			ifeatures_tile = tf.tile(ifeatures_pad, [len_crop, 1, 1, 1])

			textImgFeatures = tf.scan(self.scanFunc, [ifeatures_tile, outBoxes, cropBoxes, angles], [np.zeros((self.fix_RoiHeight, self.max_RoiWidth, channels), np.float32), np.array(0, np.int32)])
			btextImgFeatures.append(textImgFeatures[0])
			ws.append(textImgFeatures[1])

		btextImgFeatures = tf.concat(btextImgFeatures, axis=0)
		ws = tf.concat(ws, axis=0)

		return btextImgFeatures, ws
//  [python]
def __call__(self, brboxes, expand_w=20):
		paddings = tf.constant([[0, 0],[expand_w, expand_w], [expand_w, expand_w], [0, 0]])
		features_pad = tf.pad(self.features, paddings, "CONSTANT")
		features_pad = tf.expand_dims(features_pad, axis=1)
		# features_pad shape: [b, 1, h, w, c]
		nums = features_pad.shape[0]
		channels = features_pad.shape[-1]

		btextImgFeatures = []
		ws = []

		for b, rboxes in enumerate(brboxes):
			outBoxes, cropBoxes, angles = rboxes
			# outBoxes = tf.cast(tf.ceil(tf.divide(outBoxes, self.features_stride)), tf.int32)  # float div
			# cropBoxes = tf.cast(tf.ceil(tf.divide(cropBoxes, self.features_stride)), tf.int32) # float div

			outBoxes = tf.divide(outBoxes, self.features_stride)  # int div
			cropBoxes = tf.divide(cropBoxes, self.features_stride) # int div

			outBoxes_xy = outBoxes[:, :2]
			outBoxes_xy =  tf.add(outBoxes_xy, expand_w)
			outBoxes = tf.concat([outBoxes_xy, outBoxes[:, 2:]], axis=1)

			# len_crop = outBoxes.shape[0]  # error tf.stack cannot convert an unknown Dimension to a tensor: ?
			len_crop = tf.shape(outBoxes)[0]
			ifeatures_pad = features_pad[b]
			# ifeatures_tile = tf.tile(ifeatures_pad, tf.stack([len_crop, 1, 1, 1]))
			ifeatures_tile = tf.tile(ifeatures_pad, [len_crop, 1, 1, 1])

			textImgFeatures = tf.scan(self.scanFunc, [ifeatures_tile, outBoxes, cropBoxes, angles], [np.zeros((self.fix_RoiHeight, self.max_RoiWidth, channels), np.float32), np.array(0, np.int32)])
			btextImgFeatures.append(textImgFeatures[0])
			ws.append(textImgFeatures[1])

		btextImgFeatures = tf.concat(btextImgFeatures, axis=0)
		ws = tf.concat(ws, axis=0)

		return btextImgFeatures, ws
//  [python]
def _curriculum(config, step, loss_history, dependency_ops):
    """ Creates TF ops for maintaining and advancing the curriculum. """

    # assign appropriate curriculum increment value
    for case in switch(config['behavior']):
        if case('fixed_rate'):
            # fixed rate, always return same number
            increment = tf.constant(config['rate'], name='curriculum_increment')
        elif case('loss_threshold'):
            # return fixed increment if last loss is below threshold, zero otherwise
            increment_pred = tf.less(loss_history[-1], config['threshold'], name='curriculum_predicate')
            full_increment_func = lambda: tf.constant(config['rate'], name='full_curriculum_increment')
            zero_increment_func = lambda: tf.constant(0.0,            name='zero_curriculum_increment')
            increment = tf.cond(increment_pred, full_increment_func, zero_increment_func)
        elif case('loss_change'):
            # predicate for increment type
            increment_pred = tf.not_equal(loss_history[0], DUMMY_LOSS, name='curriculum_predicate')

            # increment function for when loss history is still
            def full_increment_func():
                lin_seq = tf.expand_dims(tf.linspace(0., 1., config['change_num_iterations']), 1)
                ls_matrix = tf.concat([tf.ones_like(lin_seq), lin_seq], 1)
                ls_rhs = tf.expand_dims(loss_history, 1)
                ls_slope = tf.matrix_solve_ls(ls_matrix, ls_rhs)[1, 0]

                full_increment = tf.div(config['rate'], tf.pow(tf.abs(ls_slope) + 1, config['sharpness']), name='full_curriculum_increment')

                return full_increment

            # dummy increment function for when loss history is changing rapidly
            zero_increment_func = lambda: tf.constant(0.0, name='zero_curriculum_increment')

            # final conditional increment
            increment = tf.cond(increment_pred, full_increment_func, zero_increment_func)

    # create updating op. the semantics are such that training / gradient update is first performed before the curriculum is incremented.
    with tf.control_dependencies(dependency_ops):
        update_op = tf.assign_add(step, increment, name='update_curriculum_op')

    return update_op
//  [python]
def _curriculum(config, step, loss_history, dependency_ops):
    """ Creates TF ops for maintaining and advancing the curriculum. """

    # assign appropriate curriculum increment value
    for case in switch(config['behavior']):
        if case('fixed_rate'):
            # fixed rate, always return same number
            increment = tf.constant(config['rate'], name='curriculum_increment')
        elif case('loss_threshold'):
            # return fixed increment if last loss is below threshold, zero otherwise
            increment_pred = tf.less(loss_history[-1], config['threshold'], name='curriculum_predicate')
            full_increment_func = lambda: tf.constant(config['rate'], name='full_curriculum_increment')
            zero_increment_func = lambda: tf.constant(0.0,            name='zero_curriculum_increment')
            increment = tf.cond(increment_pred, full_increment_func, zero_increment_func)
        elif case('loss_change'):
            # predicate for increment type
            increment_pred = tf.not_equal(loss_history[0], DUMMY_LOSS, name='curriculum_predicate')

            # increment function for when loss history is still
            def full_increment_func():
                lin_seq = tf.expand_dims(tf.linspace(0., 1., config['change_num_iterations']), 1)
                ls_matrix = tf.concat([tf.ones_like(lin_seq), lin_seq], 1)
                ls_rhs = tf.expand_dims(loss_history, 1)
                ls_slope = tf.matrix_solve_ls(ls_matrix, ls_rhs)[1, 0]

                full_increment = tf.divide(config['rate'], tf.pow(tf.abs(ls_slope) + 1, config['sharpness']), name='full_curriculum_increment')

                return full_increment

            # dummy increment function for when loss history is changing rapidly
            zero_increment_func = lambda: tf.constant(0.0, name='zero_curriculum_increment')

            # final conditional increment
            increment = tf.cond(increment_pred, full_increment_func, zero_increment_func)

    # create updating op. the semantics are such that training / gradient update is first performed before the curriculum is incremented.
    with tf.control_dependencies(dependency_ops):
        update_op = tf.assign_add(step, increment, name='update_curriculum_op')

    return update_op
//  [python]
def softmax(self, target, axis, name=None):
        max_axis = tf.reduce_max(target, axis, keep_dims=True)
        target_exp = tf.exp(target-max_axis)
        normalize = tf.reduce_sum(target_exp, axis, keep_dims=True)
        softmax = tf.div(target_exp, normalize, name)
        return softmax
//  [python]
def softmax(self, target, axis, name=None):
        max_axis = tf.reduce_max(target, axis, keep_dims=True)
        target_exp = tf.exp(target-max_axis)
        normalize = tf.reduce_sum(target_exp, axis, keep_dims=True)
        softmax = tf.divide(target_exp, normalize, name)
        return softmax
//  [python]
def div(x, y, name=None):
    try:
        return tf.div(x, y, name=name)
    except AttributeError:
        return tf.divide(x, y, name=name)
//  [python]
def div(x, y, name=None):
    return tf.divide(x, y, name=name)
//  [python]
def _build_net(self):
        with tf.variable_scope(self.name):
            # global_discriminator
            rand_idx1 = tf.random_shuffle(
                tf.range(tf.div(tf.size(self.globalf), tf.shape(self.globalf)[-1]))[:, None])
            rand_idx2 = tf.random_shuffle(
                tf.range(tf.div(tf.size(self.globalf), tf.shape(self.globalf)[-1]))[:, None])
            shuffle1 = tf.gather_nd(self.localf, rand_idx1)
            shuffle2 = tf.gather_nd(self.localf, rand_idx2)

            output1 = self.global_d(name="global_discriminator", localf=self.localf, globalf=self.globalf, freeze=self.freez)._build_net(reuse=self.reus)
            output2 = self.global_d(name="global_discriminator", localf=shuffle1, globalf=self.globalf, freeze=True)._build_net(reuse=tf.AUTO_REUSE)
            g_joint = tf.reduce_mean(tf.log(2.) - tf.nn.softplus(tf.negative(output1)))
            g_margi = tf.reduce_mean(tf.nn.softplus(tf.negative(output2)) + output2 - tf.log(2.))
            loss_global = (g_joint - g_margi)

            # local_discriminator
            output3 = self.local_d(name="local_discriminator", localf=self.localf, globalf=self.globalf, freeze=self.freez)._build_net(reuse=self.reus)
            output4 = self.local_d(name="local_discriminator", localf=shuffle2, globalf=self.globalf, freeze=True)._build_net(reuse=tf.AUTO_REUSE)

            l_joint_ch = tf.reduce_mean((tf.log(2.) - tf.nn.softplus(tf.negative(output3))), axis=1)
            l_margi_ch = tf.reduce_mean((tf.nn.softplus(tf.negative(output4)) + output4 - tf.log(2.)), axis=1)

            loss_local = tf.reduce_mean((l_joint_ch - l_margi_ch))

            loss_mi = loss_global + loss_local

        return loss_mi, loss_global, loss_local, g_joint, g_margi
//  [python]
def _build_net(self):
        with tf.variable_scope(self.name):
            # global_discriminator
            rand_idx1 = tf.random_shuffle(
                tf.range(tf.divide(tf.size(self.globalf), tf.shape(self.globalf)[-1]))[:, None])
            rand_idx2 = tf.random_shuffle(
                tf.range(tf.divide(tf.size(self.globalf), tf.shape(self.globalf)[-1]))[:, None])
            shuffle1 = tf.gather_nd(self.localf, rand_idx1)
            shuffle2 = tf.gather_nd(self.localf, rand_idx2)

            output1 = self.global_d(name="global_discriminator", localf=self.localf, globalf=self.globalf, freeze=self.freez)._build_net(reuse=self.reus)
            output2 = self.global_d(name="global_discriminator", localf=shuffle1, globalf=self.globalf, freeze=True)._build_net(reuse=tf.AUTO_REUSE)
            g_joint = tf.reduce_mean(tf.log(2.) - tf.nn.softplus(tf.negative(output1)))
            g_margi = tf.reduce_mean(tf.nn.softplus(tf.negative(output2)) + output2 - tf.log(2.))
            loss_global = (g_joint - g_margi)

            # local_discriminator
            output3 = self.local_d(name="local_discriminator", localf=self.localf, globalf=self.globalf, freeze=self.freez)._build_net(reuse=self.reus)
            output4 = self.local_d(name="local_discriminator", localf=shuffle2, globalf=self.globalf, freeze=True)._build_net(reuse=tf.AUTO_REUSE)

            l_joint_ch = tf.reduce_mean((tf.log(2.) - tf.nn.softplus(tf.negative(output3))), axis=1)
            l_margi_ch = tf.reduce_mean((tf.nn.softplus(tf.negative(output4)) + output4 - tf.log(2.)), axis=1)

            loss_local = tf.reduce_mean((l_joint_ch - l_margi_ch))

            loss_mi = loss_global + loss_local

        return loss_mi, loss_global, loss_local, g_joint, g_margi
//  [python]
def multi_view_att(ori_memory, att_w, dec_hidden, *args):
    bsz, max_len, rnn_hsz = args

    dec_hidden = att_w(dec_hidden)  # b*f

    ori_memory_t = tf.transpose(ori_memory, perm=[2, 0, 1])  # f*b*t
    flatten_om = tf.layers.flatten(ori_memory_t)

    beta_is = tf.exp(tf.tanh(tf.matmul(dec_hidden, flatten_om)))  # b*b*t
    beta_is = tf.reshape(beta_is, [bsz, bsz, max_len])
    beta_is = tf.transpose(beta_is, perm=[2, 0, 1])  # t*b*b

    beta_i_sum = tf.reduce_sum(beta_is, axis=0, keepdims=True)
    beta_i_sum = tf.tile(beta_i_sum, [max_len, 1, 1])
    beta_is = tf.div(beta_is, beta_i_sum)

    ori_memory_t = tf.transpose(ori_memory, perm=[1, 0, 2])
    return tf.reduce_sum(tf.matmul(beta_is, ori_memory_t), axis=0)
//  [python]
def multi_view_att(ori_memory, att_w, dec_hidden, *args):
    bsz, max_len, rnn_hsz = args

    dec_hidden = att_w(dec_hidden)  # b*f

    ori_memory_t = tf.transpose(ori_memory, perm=[2, 0, 1])  # f*b*t
    flatten_om = tf.layers.flatten(ori_memory_t)

    beta_is = tf.exp(tf.tanh(tf.matmul(dec_hidden, flatten_om)))  # b*b*t
    beta_is = tf.reshape(beta_is, [bsz, bsz, max_len])
    beta_is = tf.transpose(beta_is, perm=[2, 0, 1])  # t*b*b

    beta_i_sum = tf.reduce_sum(beta_is, axis=0, keepdims=True)
    beta_i_sum = tf.tile(beta_i_sum, [max_len, 1, 1])
    beta_is = tf.divide(beta_is, beta_i_sum)

    ori_memory_t = tf.transpose(ori_memory, perm=[1, 0, 2])
    return tf.reduce_sum(tf.matmul(beta_is, ori_memory_t), axis=0)
//  [python]
def dot_case3(dot, vec1, vec2):
    norm1 = norm_2d(vec1)
    norm2 = norm_2d(vec2)
    angle_abs = tf.math.acos(tf.div(dot, tf.multiply(norm1, norm2)))
    dir_rotation = cross_2d(vec1, vec2) / tf.abs(cross_2d(vec1, vec2))
    return angle_abs * dir_rotation
//  [python]
def dot_case3(dot, vec1, vec2):
    norm1 = norm_2d(vec1)
    norm2 = norm_2d(vec2)
    angle_abs = tf.math.acos(tf.divide(dot, tf.multiply(norm1, norm2)))
    dir_rotation = cross_2d(vec1, vec2) / tf.abs(cross_2d(vec1, vec2))
    return angle_abs * dir_rotation
//  [python]
def drop_connect(inputs, is_training, survival_prob):
  """Drop the entire conv with given survival probability."""
  # "Deep Networks with Stochastic Depth", https://arxiv.org/pdf/1603.09382.pdf
  if not is_training:
    return inputs

  # Compute tensor.
  batch_size = tf.shape(inputs)[0]
  random_tensor = survival_prob
  random_tensor += tf.random.uniform([batch_size, 1, 1, 1], dtype=inputs.dtype)
  binary_tensor = tf.floor(random_tensor)
  # Unlike conventional way that multiply survival_prob at test time, here we
  # divide survival_prob at training time, such that no addition compute is
  # needed at test time.
  output = tf.div(inputs, survival_prob) * binary_tensor
  return output
//  [python]
def drop_connect(inputs, is_training, survival_prob):
  """Drop the entire conv with given survival probability."""
  # "Deep Networks with Stochastic Depth", https://arxiv.org/pdf/1603.09382.pdf
  if not is_training:
    return inputs

  # Compute tensor.
  batch_size = tf.shape(inputs)[0]
  random_tensor = survival_prob
  random_tensor += tf.random.uniform([batch_size, 1, 1, 1], dtype=inputs.dtype)
  binary_tensor = tf.floor(random_tensor)
  # Unlike conventional way that multiply survival_prob at test time, here we
  # divide survival_prob at training time, such that no addition compute is
  # needed at test time.
  output = tf.divide(inputs, survival_prob) * binary_tensor
  return output
//  [python]
def _build_net(self):
        with tf.variable_scope(self.name):
            """ Split the global feature """
            globalf = tf.squeeze(self.input, axis=1)
            globalf = tf.squeeze(globalf, axis=1)
            re, irre = tf.split(globalf, [int(st.nfeat2/2), int(st.nfeat2/2)], axis=-1)

            """ Classifier """
            logit_re = self.DE.classifier(latent=re, freez=self.freez, reuse=self.reuse)
            loss_re = tf.reduce_mean(
                tf.nn.softmax_cross_entropy_with_logits(logits=logit_re, labels=self.label))
            pred_re = tf.argmax(tf.sigmoid(logit_re), -1)

            """ MINE """
            ridx = tf.random_shuffle(tf.range(tf.div(tf.size(globalf), tf.shape(globalf)[-1]))[:, None])
            shuffle = tf.gather_nd(re, ridx)
            output1 = self.DE.MINE(latent1=re, latent2=irre, freez=self.freez, reuse=self.reuse)
            output2 = self.DE.MINE(latent1=shuffle, latent2=irre, freez=True, reuse=tf.AUTO_REUSE)
            m_joint = tf.reduce_mean(tf.log(2.) - tf.nn.softplus(tf.negative(output1)))
            m_margi = tf.reduce_mean(tf.nn.softplus(tf.negative(output2)) + output2 - tf.log(2.))
            loss_mine = (m_joint - m_margi)

            """ Concat the class-relevant feature and class-irrelevant feautre """
            latents = tf.concat([re, irre], axis=-1)
            latents = tf.expand_dims(latents, axis=1)
            latents = tf.expand_dims(latents, axis=1)

            localfp = self.DE.spatial_deconv(latents, freez=self.freez, reuse=self.reuse)
            recon = self.DE.spectral_deconv(localfp, freez=self.freez, reuse=self.reuse)

        return loss_mine, loss_re, pred_re, recon
//  [python]
def _build_net(self):
        with tf.variable_scope(self.name):
            """ Split the global feature """
            globalf = tf.squeeze(self.input, axis=1)
            globalf = tf.squeeze(globalf, axis=1)
            re, irre = tf.split(globalf, [int(st.nfeat2/2), int(st.nfeat2/2)], axis=-1)

            """ Classifier """
            logit_re = self.DE.classifier(latent=re, freez=self.freez, reuse=self.reuse)
            loss_re = tf.reduce_mean(
                tf.nn.softmax_cross_entropy_with_logits(logits=logit_re, labels=self.label))
            pred_re = tf.argmax(tf.sigmoid(logit_re), -1)

            """ MINE """
            ridx = tf.random_shuffle(tf.range(tf.divide(tf.size(globalf), tf.shape(globalf)[-1]))[:, None])
            shuffle = tf.gather_nd(re, ridx)
            output1 = self.DE.MINE(latent1=re, latent2=irre, freez=self.freez, reuse=self.reuse)
            output2 = self.DE.MINE(latent1=shuffle, latent2=irre, freez=True, reuse=tf.AUTO_REUSE)
            m_joint = tf.reduce_mean(tf.log(2.) - tf.nn.softplus(tf.negative(output1)))
            m_margi = tf.reduce_mean(tf.nn.softplus(tf.negative(output2)) + output2 - tf.log(2.))
            loss_mine = (m_joint - m_margi)

            """ Concat the class-relevant feature and class-irrelevant feautre """
            latents = tf.concat([re, irre], axis=-1)
            latents = tf.expand_dims(latents, axis=1)
            latents = tf.expand_dims(latents, axis=1)

            localfp = self.DE.spatial_deconv(latents, freez=self.freez, reuse=self.reuse)
            recon = self.DE.spectral_deconv(localfp, freez=self.freez, reuse=self.reuse)

        return loss_mine, loss_re, pred_re, recon
//  [python]
def __init__(
            self,
            layer=None,
            act=tf.identity,
            epsilon=1e-5,
            scale_init=tf.truncated_normal_initializer(mean=1.0, stddev=0.02),
            offset_init=tf.constant_initializer(0.0),
            name='instan_norm',
    ):
        Layer.__init__(self, name=name)
        self.inputs = layer.outputs
        print("  [TL] InstanceNormLayer %s: epsilon:%f act:%s" % (self.name, epsilon, act.__name__))

        with tf.variable_scope(name) as vs:
            mean, var = tf.nn.moments(self.inputs, [1, 2], keep_dims=True)
            scale = tf.get_variable('scale', [self.inputs.get_shape()[-1]], initializer=tf.truncated_normal_initializer(mean=1.0, stddev=0.02), dtype=D_TYPE)
            offset = tf.get_variable('offset', [self.inputs.get_shape()[-1]], initializer=tf.constant_initializer(0.0), dtype=D_TYPE)
            self.outputs = scale * tf.div(self.inputs - mean, tf.sqrt(var + epsilon)) + offset
            self.outputs = act(self.outputs)
            variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)

        self.all_layers = list(layer.all_layers)
        self.all_params = list(layer.all_params)
        self.all_drop = dict(layer.all_drop)
        self.all_layers.extend([self.outputs])
        self.all_params.extend(variables)
//  [python]
def __init__(
            self,
            layer=None,
            act=tf.identity,
            epsilon=1e-5,
            scale_init=tf.truncated_normal_initializer(mean=1.0, stddev=0.02),
            offset_init=tf.constant_initializer(0.0),
            name='instan_norm',
    ):
        Layer.__init__(self, name=name)
        self.inputs = layer.outputs
        print("  [TL] InstanceNormLayer %s: epsilon:%f act:%s" % (self.name, epsilon, act.__name__))

        with tf.variable_scope(name) as vs:
            mean, var = tf.nn.moments(self.inputs, [1, 2], keep_dims=True)
            scale = tf.get_variable('scale', [self.inputs.get_shape()[-1]], initializer=tf.truncated_normal_initializer(mean=1.0, stddev=0.02), dtype=D_TYPE)
            offset = tf.get_variable('offset', [self.inputs.get_shape()[-1]], initializer=tf.constant_initializer(0.0), dtype=D_TYPE)
            self.outputs = scale * tf.divide(self.inputs - mean, tf.sqrt(var + epsilon)) + offset
            self.outputs = act(self.outputs)
            variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)

        self.all_layers = list(layer.all_layers)
        self.all_params = list(layer.all_params)
        self.all_drop = dict(layer.all_drop)
        self.all_layers.extend([self.outputs])
        self.all_params.extend(variables)
//  [python]
def softmax(self, target, axis, name=None):
        max_axis = tf.reduce_max(target, axis, keepdims=True)
        target_exp = tf.exp(target - max_axis)
        normalize = tf.reduce_sum(target_exp, axis, keepdims=True)
        softmax = tf.div(target_exp, normalize, name)
        return softmax
//  [python]
def softmax(self, target, axis, name=None):
        max_axis = tf.reduce_max(target, axis, keepdims=True)
        target_exp = tf.exp(target - max_axis)
        normalize = tf.reduce_sum(target_exp, axis, keepdims=True)
        softmax = tf.divide(target_exp, normalize, name)
        return softmax
//  [python]
def Norm_brightness(self,input_tensor):

        input_tensor1 = tf.layers.average_pooling2d(input_tensor,[3,3],strides=1, padding='SAME')#denoise
        
        bright_min = tf.reduce_min(input_tensor1)
        bright_max = tf.reduce_max(input_tensor1)
        grad_norm = (tf.div((input_tensor - bright_min), (bright_max - bright_min + 0.0001)))
        return grad_norm
//  [python]
def Norm_brightness(self,input_tensor):

        input_tensor1 = tf.layers.average_pooling2d(input_tensor,[3,3],strides=1, padding='SAME')#denoise
        
        bright_min = tf.reduce_min(input_tensor1)
        bright_max = tf.reduce_max(input_tensor1)
        grad_norm = tf.divide((input_tensor - bright_min), (bright_max - bright_min + 0.0001))
        return grad_norm
//  [python]
def _secondary_loss(config, dssps, targets, masks, group_filter):
    """ Computes reduced cross entropy loss between predicted secondary 
        structures and targets. """

    # XXX THIS SECTION IS NOT CURRENTLY FUNCTIONAL

    # flatten before cross entropy computation
    my_dssps = tf.identity(dssps, 'my_linear_dssps')

    flattened_dssps   = tf.reshape(dssps,   [-1, NUM_DSSPS]) # [NUM_STEPS x BATCH_SIZE, NUM_DSSPS]
    flattened_targets = tf.reshape(targets, [-1])            # [NUM_STEPS x BATCH_SIZE]

    # compute softmax cross entropy from raw activations, and turn nans into zeroes for predictions past sequence lengths
    flattened_sces = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=flattened_targets, logits=flattened_dssps)
    flattened_sces = tf.where(tf.is_nan(flattened_sces), tf.zeros_like(flattened_sces), flattened_sces)
                     # [NUM_STEPS x BATCH_SIZE] 

    # reshape, reduce, and normalize
    eff_num_stepss = effective_steps(masks, config['num_edge_residues'])                   # [BATCH_SIZE]
    sces = tf.reshape(flattened_sces, [tf.shape(dssps)[0], -1])                            # [NUM_STEPS, BATCH_SIZE] 
    sces_reduced = tf.reduce_sum(sces, 0)                                                  # [BATCH_SIZE]
    sces_normalized = tf.div(sces_reduced, eff_num_stepss, name='secondary_cross_entropy') # [BATCH_SIZE]

    # reduce cross_entropy based on normalization behavior
    loss = _reduce_loss(sces_normalized, masks, group_filter, 
                        config['secondary_normalization'], config['num_edge_residues'], name='secondary_loss')

    # add to relevant collections for summaries, etc.
    if config['log_model_summaries']: 
        tf.add_to_collection(config['name'] + '_scess',            sces_normalized) # never filters by group
        tf.add_to_collection(config['name'] + '_secondary_losses', loss)

    return loss
//  [python]
def _secondary_loss(config, dssps, targets, masks, group_filter):
    """ Computes reduced cross entropy loss between predicted secondary 
        structures and targets. """

    # XXX THIS SECTION IS NOT CURRENTLY FUNCTIONAL

    # flatten before cross entropy computation
    my_dssps = tf.identity(dssps, 'my_linear_dssps')

    flattened_dssps   = tf.reshape(dssps,   [-1, NUM_DSSPS]) # [NUM_STEPS x BATCH_SIZE, NUM_DSSPS]
    flattened_targets = tf.reshape(targets, [-1])            # [NUM_STEPS x BATCH_SIZE]

    # compute softmax cross entropy from raw activations, and turn nans into zeroes for predictions past sequence lengths
    flattened_sces = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=flattened_targets, logits=flattened_dssps)
    flattened_sces = tf.where(tf.is_nan(flattened_sces), tf.zeros_like(flattened_sces), flattened_sces)
                     # [NUM_STEPS x BATCH_SIZE] 

    # reshape, reduce, and normalize
    eff_num_stepss = effective_steps(masks, config['num_edge_residues'])                   # [BATCH_SIZE]
    sces = tf.reshape(flattened_sces, [tf.shape(dssps)[0], -1])                            # [NUM_STEPS, BATCH_SIZE] 
    sces_reduced = tf.reduce_sum(sces, 0)                                                  # [BATCH_SIZE]
    sces_normalized = tf.divide(sces_reduced, eff_num_stepss, name='secondary_cross_entropy') # [BATCH_SIZE]

    # reduce cross_entropy based on normalization behavior
    loss = _reduce_loss(sces_normalized, masks, group_filter, 
                        config['secondary_normalization'], config['num_edge_residues'], name='secondary_loss')

    # add to relevant collections for summaries, etc.
    if config['log_model_summaries']: 
        tf.add_to_collection(config['name'] + '_scess',            sces_normalized) # never filters by group
        tf.add_to_collection(config['name'] + '_secondary_losses', loss)

    return loss
//  [python]
def gradient_n(self,input_tensor, direction):
        smooth_kernel_x = tf.reshape(tf.constant([[0, 0], [-1, 1]], tf.float32), [2, 2, 1, 1])
        smooth_kernel_y = tf.transpose(smooth_kernel_x, [1, 0, 2, 3])
        if direction == "x":
            kernel = smooth_kernel_x
        elif direction == "y":
            kernel = smooth_kernel_y
        gradient_orig = tf.abs(tf.nn.conv2d(input_tensor, kernel, strides=[1, 1, 1, 1], padding='SAME'))
        grad_min = tf.reduce_min(gradient_orig)
        grad_max = tf.reduce_max(gradient_orig)
        grad_norm = tf.div((gradient_orig - grad_min), (grad_max - grad_min + 0.0001))
        return grad_norm
//  [python]
def gradient_n(self,input_tensor, direction):
        smooth_kernel_x = tf.reshape(tf.constant([[0, 0], [-1, 1]], tf.float32), [2, 2, 1, 1])
        smooth_kernel_y = tf.transpose(smooth_kernel_x, [1, 0, 2, 3])
        if direction == "x":
            kernel = smooth_kernel_x
        elif direction == "y":
            kernel = smooth_kernel_y
        gradient_orig = tf.abs(tf.nn.conv2d(input_tensor, kernel, strides=[1, 1, 1, 1], padding='SAME'))
        grad_min = tf.reduce_min(gradient_orig)
        grad_max = tf.reduce_max(gradient_orig)
        grad_norm = tf.divide((gradient_orig - grad_min), (grad_max - grad_min + 0.0001))
        return grad_norm
//  [python]
def fprop(self, x, **kwargs):
        axis = [1, 2, 3]
        mean = reduce_mean(x, axis=axis, keepdims=True)
        variance = reduce_mean(tf.square(x), axis=axis, keepdims=True) - tf.square(mean)
        variance = tf.nn.relu(variance)
        stddev = tf.sqrt(variance)

        num_pixels = reduce_prod(tf.shape(x)[1:])

        min_stddev = tf.rsqrt(tf.to_float(num_pixels))
        pixel_value_scale = tf.maximum(stddev, min_stddev)
        pixel_value_offset = mean

        x = tf.subtract(x, pixel_value_offset)
        x = tf.div(x, pixel_value_scale)
        return x
//  [python]
def fprop(self, x, **kwargs):
        axis = [1, 2, 3]
        mean = reduce_mean(x, axis=axis, keepdims=True)
        variance = reduce_mean(tf.square(x), axis=axis, keepdims=True) - tf.square(mean)
        variance = tf.nn.relu(variance)
        stddev = tf.sqrt(variance)

        num_pixels = reduce_prod(tf.shape(x)[1:])

        min_stddev = tf.rsqrt(tf.to_float(num_pixels))
        pixel_value_scale = tf.maximum(stddev, min_stddev)
        pixel_value_offset = mean

        x = tf.subtract(x, pixel_value_offset)
        x = tf.divide(x, pixel_value_scale)
        return x
//  [python]
def update_for_u(self, u_pre, V, e):
        a = tf.div(tf.exp(e), tf.reduce_sum(tf.exp(e)))
        o = tf.matmul(a, self.x)
        u = tf.add(tf.tanh(tf.matmul(u_pre, V)), o)
        return u
//  [python]
def update_for_u(self, u_pre, V, e):
        a = tf.divide(tf.exp(e), tf.reduce_sum(tf.exp(e)))
        o = tf.matmul(a, self.x)
        u = tf.add(tf.tanh(tf.matmul(u_pre, V)), o)
        return u
//  [python]
def apply_gradients(self, gradvars, *args, **kwargs):
        if self._eta is not None:
            v_list = [tf.norm(tensor=v, ord=2) for _, v in gradvars]
            g_list = [tf.norm(tensor=g, ord=2) if g is not None else 0.0
                      for g, _ in gradvars]
            v_norms = tf.stack(v_list)
            g_norms = tf.stack(g_list)
            zeds = tf.zeros_like(v_norms)
            cond = tf.logical_and(
                tf.not_equal(v_norms, zeds),
                tf.not_equal(g_norms, zeds))
            true_vals = tf.scalar_mul(self._eta, tf.div(v_norms, g_norms))
            false_vals = tf.fill(tf.shape(v_norms), self._epsilon)
            larc_local_lr = tf.where(cond, true_vals, false_vals)
            if self._clip:
                ones = tf.ones_like(v_norms)
                lr = tf.fill(tf.shape(v_norms), self._learning_rate)
                larc_local_lr = tf.minimum(tf.div(larc_local_lr, lr), ones)
            gradvars = [(tf.multiply(larc_local_lr[i], g), v)
                        if g is not None else (None, v)
                        for i, (g, v) in enumerate(gradvars) ]
        return self._optimizer.apply_gradients(gradvars, *args, **kwargs)
//  [python]
def apply_gradients(self, gradvars, *args, **kwargs):
        if self._eta is not None:
            v_list = [tf.norm(tensor=v, ord=2) for _, v in gradvars]
            g_list = [tf.norm(tensor=g, ord=2) if g is not None else 0.0
                      for g, _ in gradvars]
            v_norms = tf.stack(v_list)
            g_norms = tf.stack(g_list)
            zeds = tf.zeros_like(v_norms)
            cond = tf.logical_and(
                tf.not_equal(v_norms, zeds),
                tf.not_equal(g_norms, zeds))
            true_vals = tf.scalar_mul(self._eta, tf.divide(v_norms, g_norms))
            false_vals = tf.fill(tf.shape(v_norms), self._epsilon)
            larc_local_lr = tf.where(cond, true_vals, false_vals)
            if self._clip:
                ones = tf.ones_like(v_norms)
                lr = tf.fill(tf.shape(v_norms), self._learning_rate)
                larc_local_lr = tf.minimum(tf.divide(larc_local_lr, lr), ones)
            gradvars = [(tf.multiply(larc_local_lr[i], g), v)
                        if g is not None else (None, v)
                        for i, (g, v) in enumerate(gradvars) ]
        return self._optimizer.apply_gradients(gradvars, *args, **kwargs)
//  [python]
def CameraJacobianMatrix(self,x,y,Z,fx,fy,name=None):
        with tf.name_scope(name):
            xy_z2= tf.multiply(x,y)
            xx_z2= -1.0-tf.square(x)
            x_z2 = tf.div(x,Z)
            yy_z2= 1.0+tf.square(y)
            y_z2 = tf.div(y,Z)
            _Z   = 1.0/Z
            zeros= tf.zeros(xy_z2.get_shape())
            dx   = tf.expand_dims(fx,axis=-1)*tf.stack([xy_z2, xx_z2, y,-_Z,zeros,x_z2],axis=2)
            dy   = tf.expand_dims(fy,axis=-1)*tf.stack([yy_z2,-xy_z2,-x,zeros,-_Z,y_z2],axis=2)
            jacobianMatrixGeometry=tf.stack([dx,dy],axis=2)
        return jacobianMatrixGeometry
//  [python]
def CameraJacobianMatrix(self,x,y,Z,fx,fy,name=None):
        with tf.name_scope(name):
            xy_z2= tf.multiply(x,y)
            xx_z2= -1.0-tf.square(x)
            x_z2 = tf.divide(x,Z)
            yy_z2= 1.0+tf.square(y)
            y_z2 = tf.divide(y,Z)
            _Z   = 1.0/Z
            zeros= tf.zeros(xy_z2.get_shape())
            dx   = tf.expand_dims(fx,axis=-1)*tf.stack([xy_z2, xx_z2, y,-_Z,zeros,x_z2],axis=2)
            dy   = tf.expand_dims(fy,axis=-1)*tf.stack([yy_z2,-xy_z2,-x,zeros,-_Z,y_z2],axis=2)
            jacobianMatrixGeometry=tf.stack([dx,dy],axis=2)
        return jacobianMatrixGeometry
//  [python]
def SAD(x,y,reuse = False, dimension_same = False):
    if reuse:
        tf.get_variable_scope().reuse_variables()
    with tf.name_scope('SAD'):
        
        if dimension_same:

            A = tf.matmul(x,tf.transpose(y))  #像素点数*像素点数
            A = tf.diag_part(A) #像素点数*1
            B = tf.matmul(x,tf.transpose(x))
            B = tf.diag_part(B) 
            B = pow(B, 0.5)
            C = tf.matmul(y,tf.transpose(y))
            C = tf.diag_part(C)
            C = pow(C, 0.5)
            defen = A/(B*C)#矩阵对应元素运算    
            print("defen")
            print(defen)

        else: 
            A = tf.matmul(x,y)#矩阵相乘
            B = tf.norm(x,ord = 2)#矩阵的二范数
            C = tf.norm(y,ord = 2)
            defen = tf.div(A,B*C)

        defen = tf.acos(defen) #arccos ~[0-π]
        #sad_normalize = 1 - tf.div(defen , math.pi)  #  归一化
        sad_normalize = tf.div(defen , math.pi)
        spectral_loss = tf.div(sad_normalize,batch_size)
        return spectral_loss
//  [python]
def SAD(x,y,reuse = False, dimension_same = False):
    if reuse:
        tf.get_variable_scope().reuse_variables()
    with tf.name_scope('SAD'):
        
        if dimension_same:

            A = tf.matmul(x,tf.transpose(y))  #像素点数*像素点数
            A = tf.diag_part(A) #像素点数*1
            B = tf.matmul(x,tf.transpose(x))
            B = tf.diag_part(B) 
            B = pow(B, 0.5)
            C = tf.matmul(y,tf.transpose(y))
            C = tf.diag_part(C)
            C = pow(C, 0.5)
            defen = tf.divide(A,(B*C))#矩阵对应元素运算    
            print("defen")
            print(defen)

        else: 
            A = tf.matmul(x,y)#矩阵相乘
            B = tf.norm(x,ord = 2)#矩阵的二范数
            C = tf.norm(y,ord = 2)
            defen = tf.divide(A,B*C)

        defen = tf.acos(defen) #arccos ~[0-π]
        #sad_normalize = 1 - tf.divide(defen , math.pi)  #  归一化
        sad_normalize = tf.divide(defen , math.pi)
        spectral_loss = tf.divide(sad_normalize,batch_size)
        return spectral_loss
//  [python]
def compute_loss(self, predictions, labels, num_labels):
		"""
		Mean squared error
		"""
		with tf.name_scope('loss'):
			loss = tf.div(tf.reduce_sum(tf.square(tf.subtract(predictions,labels))),num_labels)
			return loss
//  [python]
def compute_loss(self, predictions, labels, num_labels):
		"""
		Mean squared error
		"""
		with tf.name_scope('loss'):
			loss = tf.divide(tf.reduce_sum(tf.square(tf.subtract(predictions,labels))),num_labels)
			return loss
//  [python]
def crop_features(feature, out_size):
    """Crop the center of a feature map
    Args:
    feature: Feature map to crop
    out_size: Size of the output feature map
    Returns:
    Tensor that performs the cropping
    """
    # Crop the center of a feature map
    up_size = tf.shape(feature)
    ini_w = tf.div(tf.subtract(up_size[1], out_size[1]), 2)
    ini_h = tf.div(tf.subtract(up_size[2], out_size[2]), 2)
    slice_input = tf.slice(feature, (0, ini_w, ini_h, 0), (-1, out_size[1], out_size[2], -1))
    return tf.reshape(slice_input, [int(feature.get_shape()[0]), out_size[1], out_size[2], int(feature.get_shape()[3])])
//  [python]
def crop_features(feature, out_size):
    """Crop the center of a feature map
    Args:
    feature: Feature map to crop
    out_size: Size of the output feature map
    Returns:
    Tensor that performs the cropping
    """
    # Crop the center of a feature map
    up_size = tf.shape(feature)
    ini_w = tf.divide(tf.subtract(up_size[1], out_size[1]), 2)
    ini_h = tf.divide(tf.subtract(up_size[2], out_size[2]), 2)
    slice_input = tf.slice(feature, (0, ini_w, ini_h, 0), (-1, out_size[1], out_size[2], -1))
    return tf.reshape(slice_input, [int(feature.get_shape()[0]), out_size[1], out_size[2], int(feature.get_shape()[3])])
//  [python]
def masked_within_block_local_attention_1d(q, k, v, block_length=64, name=None):
  """Attention to the source and a neighborhood to the left within a block.

  The sequence is divided into blocks of length block_length. Attention for a
  given query position can only see memory positions less than or equal to the
  query position in the corresponding block.

  Args:
    q: a Tensor with shape [batch, heads, length, depth_k]
    k: a Tensor with shape [batch, heads, length, depth_k]
    v: a Tensor with shape [batch, heads, length, depth_v]
    block_length: an integer
    name: an optional string

  Returns:
    a Tensor of shape [batch, heads, length, depth_v]
  """
  with tf.variable_scope(
      name, default_name="within_local_attention_1d", values=[q, k, v]):
    batch, heads, length, depth_k = common_layers.shape_list(q)
    depth_v = common_layers.shape_list(v)[-1]
    if isinstance(block_length, tf.Tensor):
      const = contrib.util().constant_value(block_length)
      if const is not None:
        block_length = int(const)

    # Pad query, key, value to ensure multiple of block length.
    original_length = length
    padding_size = tf.mod(-length, block_length)
    length += padding_size
    padding = [[0, 0], [0, 0], [0, padding_size], [0, 0]]
    q = tf.pad(q, padding)
    k = tf.pad(k, padding)
    v = tf.pad(v, padding)

    # Compute attention for all subsequent query blocks.
    num_blocks = tf.div(length, block_length)
    q = tf.reshape(q, [batch, heads, num_blocks, block_length, depth_k])
    k = tf.reshape(k, [batch, heads, num_blocks, block_length, depth_k])
    v = tf.reshape(v, [batch, heads, num_blocks, block_length, depth_v])
    # [batch, heads, num_blocks, block_length, block_length]
    attention = tf.matmul(q, k, transpose_b=True)
    attention += tf.reshape(attention_bias_lower_triangle(block_length),
                            [1, 1, 1, block_length, block_length])
    attention = tf.nn.softmax(attention)
    # [batch, heads, num_blocks, block_length, depth_v]
    output = tf.matmul(attention, v)
    output = tf.reshape(output, [batch, heads, -1, depth_v])

    # Remove the padding if introduced.
    output = tf.slice(output, [0, 0, 0, 0], [-1, -1, original_length, -1])
    output.set_shape([None if isinstance(dim, tf.Tensor) else dim for dim in
                      (batch, heads, length, depth_v)])
    return output
//  [python]
def masked_within_block_local_attention_1d(q, k, v, block_length=64, name=None):
  """Attention to the source and a neighborhood to the left within a block.

  The sequence is divided into blocks of length block_length. Attention for a
  given query position can only see memory positions less than or equal to the
  query position in the corresponding block.

  Args:
    q: a Tensor with shape [batch, heads, length, depth_k]
    k: a Tensor with shape [batch, heads, length, depth_k]
    v: a Tensor with shape [batch, heads, length, depth_v]
    block_length: an integer
    name: an optional string

  Returns:
    a Tensor of shape [batch, heads, length, depth_v]
  """
  with tf.variable_scope(
      name, default_name="within_local_attention_1d", values=[q, k, v]):
    batch, heads, length, depth_k = common_layers.shape_list(q)
    depth_v = common_layers.shape_list(v)[-1]
    if isinstance(block_length, tf.Tensor):
      const = contrib.util().constant_value(block_length)
      if const is not None:
        block_length = int(const)

    # Pad query, key, value to ensure multiple of block length.
    original_length = length
    padding_size = tf.mod(-length, block_length)
    length += padding_size
    padding = [[0, 0], [0, 0], [0, padding_size], [0, 0]]
    q = tf.pad(q, padding)
    k = tf.pad(k, padding)
    v = tf.pad(v, padding)

    # Compute attention for all subsequent query blocks.
    num_blocks = tf.divide(length, block_length)
    q = tf.reshape(q, [batch, heads, num_blocks, block_length, depth_k])
    k = tf.reshape(k, [batch, heads, num_blocks, block_length, depth_k])
    v = tf.reshape(v, [batch, heads, num_blocks, block_length, depth_v])
    # [batch, heads, num_blocks, block_length, block_length]
    attention = tf.matmul(q, k, transpose_b=True)
    attention += tf.reshape(attention_bias_lower_triangle(block_length),
                            [1, 1, 1, block_length, block_length])
    attention = tf.nn.softmax(attention)
    # [batch, heads, num_blocks, block_length, depth_v]
    output = tf.matmul(attention, v)
    output = tf.reshape(output, [batch, heads, -1, depth_v])

    # Remove the padding if introduced.
    output = tf.slice(output, [0, 0, 0, 0], [-1, -1, original_length, -1])
    output.set_shape([None if isinstance(dim, tf.Tensor) else dim for dim in
                      (batch, heads, length, depth_v)])
    return output
//  [python]
def drop_connect(inputs, is_training, drop_connect_rate):
  """Apply drop connect."""
  if not is_training:
    return inputs

  # Compute keep_prob
  # TODO(tanmingxing): add support for training progress.
  keep_prob = 1.0 - drop_connect_rate

  # Compute drop_connect tensor
  batch_size = tf.shape(inputs)[0]
  random_tensor = keep_prob
  random_tensor += tf.random_uniform([batch_size, 1, 1, 1], dtype=inputs.dtype)
  binary_tensor = tf.floor(random_tensor)
  output = tf.div(inputs, keep_prob) * binary_tensor
  return output
//  [python]
def drop_connect(inputs, is_training, drop_connect_rate):
  """Apply drop connect."""
  if not is_training:
    return inputs

  # Compute keep_prob
  # TODO(tanmingxing): add support for training progress.
  keep_prob = 1.0 - drop_connect_rate

  # Compute drop_connect tensor
  batch_size = tf.shape(inputs)[0]
  random_tensor = keep_prob
  random_tensor += tf.random_uniform([batch_size, 1, 1, 1], dtype=inputs.dtype)
  binary_tensor = tf.floor(random_tensor)
  output = tf.divide(inputs, keep_prob) * binary_tensor
  return output
//  [python]
def project(self, points):
		points = self.transform(points)
		points = points + 1e-8

		xs = tf.div(points[:, 0], points[:, 2])
		ys = tf.div(points[:, 1], points[:, 2])
		us = self.fl_x * xs + self.cx
		vs = self.fl_y * ys + self.cy
		#vs = 480 - vs
		res = tf.stack([us, vs], axis=1)

		return res
//  [python]
def project(self, points):
		points = self.transform(points)
		points = points + 1e-8

		xs = tf.divide(points[:, 0], points[:, 2])
		ys = tf.divide(points[:, 1], points[:, 2])
		us = self.fl_x * xs + self.cx
		vs = self.fl_y * ys + self.cy
		#vs = 480 - vs
		res = tf.stack([us, vs], axis=1)

		return res
//  [python]
def _softmax_with_mask(self, um_batch, m_mask_batch):
        exp_um_batch = tf.exp(um_batch)  # [N, M]
        masked_batch = exp_um_batch * m_mask_batch  # [N, M]
        sum_2d_batch = tf.expand_dims(tf.reduce_sum(masked_batch, 1), -1)  # [N, 1]
        p_batch = tf.div(masked_batch, sum_2d_batch, name='p')  # [N, M]
        return p_batch