jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def tf_matmul_separate(A,B):
  batch,a_h,a_w,_,_ = A.get_shape()
  if batch*a_h*a_w < 80000:
      return tf.matmul(A,B)
  w_half = tf.cast(tf.divide(a_w,2),tf.int32)
  h_half = tf.cast(tf.divide(a_h,2),tf.int32)
  A_1 = A[:,:h_half,:w_half,:,:]
  A_2 = A[:,:h_half,w_half:,:,:]
  A_3 = A[:,h_half:,:w_half,:,:]
  A_4 = A[:,h_half:,w_half:,:,:]
  
  B_1 = B[:,:h_half,:w_half,:,:]
  B_2 = B[:,:h_half,w_half:,:,:]
  B_3 = B[:,h_half:,:w_half,:,:]
  B_4 = B[:,h_half:,w_half:,:,:]
  
  res_1 = tf_matmul_separate(A_1,B_1)
  res_2 = tf_matmul_separate(A_2,B_2)
  res_3 = tf_matmul_separate(A_3,B_3)
  res_4 = tf_matmul_separate(A_4,B_4)
  tmp_up = tf.concat([res_1,res_2],2)
  tmp_down = tf.concat([res_3,res_4],2)
  final_res = tf.concat([tmp_up,tmp_down],1)
  return final_res
//  [python]
def _setup_sparsity(self):
    begin_step = self._spec.sparsity_function_begin_step
    end_step = self._spec.sparsity_function_end_step
    initial_sparsity = self._spec.initial_sparsity
    target_sparsity = self._spec.target_sparsity
    exponent = self._spec.sparsity_function_exponent

    with tf.name_scope(self._spec.name):
      p = tf.minimum(
          1.0,
          tf.maximum(
              0.0,
              tf.div(
                  tf.cast(self._global_step - begin_step, tf.float32),
                  end_step - begin_step)))
      sparsity = tf.add(
          tf.multiply(initial_sparsity - target_sparsity,
                      tf.pow(1 - p, exponent)),
          target_sparsity,
          name='sparsity')

    return sparsity
//  [python]
def _setup_sparsity(self):
    begin_step = self._spec.sparsity_function_begin_step
    end_step = self._spec.sparsity_function_end_step
    initial_sparsity = self._spec.initial_sparsity
    target_sparsity = self._spec.target_sparsity
    exponent = self._spec.sparsity_function_exponent

    with tf.name_scope(self._spec.name):
      p = tf.minimum(
          1.0,
          tf.maximum(
              0.0,
              tf.divide(
                  tf.cast(self._global_step - begin_step, tf.float32),
                  end_step - begin_step)))
      sparsity = tf.add(
          tf.multiply(initial_sparsity - target_sparsity,
                      tf.pow(1 - p, exponent)),
          target_sparsity,
          name='sparsity')

    return sparsity
//  [python]
def _get_group_sparsity_raw(self, group_name):
    target_sparsity = self._group_sparsity_map_raw.get(group_name)

    if not target_sparsity:
      return self._sparsity

    return tf.multiply(self._sparsity,
                       tf.div(target_sparsity, self._spec.target_sparsity))
//  [python]
def _get_group_sparsity_raw(self, group_name):
    target_sparsity = self._group_sparsity_map_raw.get(group_name)

    if not target_sparsity:
      return self._sparsity

    return tf.multiply(self._sparsity,
                       tf.divide(target_sparsity, self._spec.target_sparsity))
//  [python]
def _get_sparsity(self, weight_name):
    """Returns target sparsity for the given layer/weight name."""
    target_sparsity = [
        sparsity for regexp, sparsity in self._weight_sparsity_map.items()
        if regexp.search(weight_name)
    ]
    if not target_sparsity:
      return self._sparsity

    if len(target_sparsity) > 1:
      raise ValueError('Multiple matches in weight_sparsity_map for weight %s' %
                       weight_name)
    # TODO(suyoggupta): This will work when initial_sparsity = 0. Generalize
    # to handle other cases as well.
    return tf.multiply(self._sparsity,
                       tf.div(target_sparsity[0], self._spec.target_sparsity))
//  [python]
def _get_sparsity(self, weight_name):
    """Returns target sparsity for the given layer/weight name."""
    target_sparsity = [
        sparsity for regexp, sparsity in self._weight_sparsity_map.items()
        if regexp.search(weight_name)
    ]
    if not target_sparsity:
      return self._sparsity

    if len(target_sparsity) > 1:
      raise ValueError('Multiple matches in weight_sparsity_map for weight %s' %
                       weight_name)
    # TODO(suyoggupta): This will work when initial_sparsity = 0. Generalize
    # to handle other cases as well.
    return tf.multiply(self._sparsity,
                       tf.divide(target_sparsity[0], self._spec.target_sparsity))
//  [python]
def _get_group_sparsity(self, group_name):
    """Returns target sparsity for the given group name."""
    target_sparsity = [
        sparsity for regexp, sparsity in self._group_sparsity_map.items()
        if regexp.search(group_name)
    ]
    if not target_sparsity:
      return self._sparsity

    if len(target_sparsity) > 1:
      raise ValueError('Multiple matches in group_sparsity_map for group %s' %
                       group_name)
    # TODO(suyoggupta): This will work when initial_sparsity = 0. Generalize
    # to handle other cases as well.
    return tf.multiply(self._sparsity,
                       tf.div(target_sparsity[0], self._spec.target_sparsity))
//  [python]
def _get_group_sparsity(self, group_name):
    """Returns target sparsity for the given group name."""
    target_sparsity = [
        sparsity for regexp, sparsity in self._group_sparsity_map.items()
        if regexp.search(group_name)
    ]
    if not target_sparsity:
      return self._sparsity

    if len(target_sparsity) > 1:
      raise ValueError('Multiple matches in group_sparsity_map for group %s' %
                       group_name)
    # TODO(suyoggupta): This will work when initial_sparsity = 0. Generalize
    # to handle other cases as well.
    return tf.multiply(self._sparsity,
                       tf.divide(target_sparsity[0], self._spec.target_sparsity))
//  [python]
def tf_Normalize(tensor):
    Length = tf.sqrt(tf.reduce_sum(tf.square(tensor), axis = -1, keep_dims=True))
    return tf.div(tensor, Length)
//  [python]
def tf_Normalize(tensor):
    Length = tf.sqrt(tf.reduce_sum(tf.square(tensor), axis = -1, keep_dims=True))
    return tf.divide(tensor, Length)
//  [python]
def _gather_clone_loss(clone, num_clones, regularization_losses):
  """Gather the loss for a single clone.

  Args:
    clone: A Clone namedtuple.
    num_clones: The number of clones being deployed.
    regularization_losses: Possibly empty list of regularization_losses
      to add to the clone losses.

  Returns:
    A tensor for the total loss for the clone.  Can be None.
  """
  # The return value.
  sum_loss = None
  # Individual components of the loss that will need summaries.
  clone_loss = None
  regularization_loss = None
  # Compute and aggregate losses on the clone device.
  with tf.device(clone.device):
    all_losses = []
    clone_losses = tf.get_collection(tf.GraphKeys.LOSSES, clone.scope)
    if clone_losses:
      clone_loss = tf.add_n(clone_losses, name='clone_loss')
      if num_clones > 1:
        clone_loss = tf.div(clone_loss, 1.0 * num_clones,
                            name='scaled_clone_loss')
      all_losses.append(clone_loss)
    if regularization_losses:
      regularization_loss = tf.add_n(regularization_losses,
                                     name='regularization_loss')
      all_losses.append(regularization_loss)
    if all_losses:
      sum_loss = tf.add_n(all_losses)
  # Add the summaries out of the clone device block.
  if clone_loss is not None:
    tf.summary.scalar('/'.join(filter(None,
                                      ['Losses', clone.scope, 'clone_loss'])),
                      clone_loss)
  if regularization_loss is not None:
    tf.summary.scalar('Losses/regularization_loss', regularization_loss)
  return sum_loss
//  [python]
def _gather_clone_loss(clone, num_clones, regularization_losses):
  """Gather the loss for a single clone.

  Args:
    clone: A Clone namedtuple.
    num_clones: The number of clones being deployed.
    regularization_losses: Possibly empty list of regularization_losses
      to add to the clone losses.

  Returns:
    A tensor for the total loss for the clone.  Can be None.
  """
  # The return value.
  sum_loss = None
  # Individual components of the loss that will need summaries.
  clone_loss = None
  regularization_loss = None
  # Compute and aggregate losses on the clone device.
  with tf.device(clone.device):
    all_losses = []
    clone_losses = tf.get_collection(tf.GraphKeys.LOSSES, clone.scope)
    if clone_losses:
      clone_loss = tf.add_n(clone_losses, name='clone_loss')
      if num_clones > 1:
        clone_loss = tf.divide(clone_loss, 1.0 * num_clones,
                            name='scaled_clone_loss')
      all_losses.append(clone_loss)
    if regularization_losses:
      regularization_loss = tf.add_n(regularization_losses,
                                     name='regularization_loss')
      all_losses.append(regularization_loss)
    if all_losses:
      sum_loss = tf.add_n(all_losses)
  # Add the summaries out of the clone device block.
  if clone_loss is not None:
    tf.summary.scalar('/'.join(filter(None,
                                      ['Losses', clone.scope, 'clone_loss'])),
                      clone_loss)
  if regularization_loss is not None:
    tf.summary.scalar('Losses/regularization_loss', regularization_loss)
  return sum_loss
//  [python]
top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k)
top_k_xvals = tf.truediv(1.0, top_k_xvals)
x_sums = tf.expand_dims(tf.reduce_sum(top_k_xvals, 1),1)
x_sums_repeated = tf.matmul(x_sums,tf.ones([1, k], tf.float32))
x_val_weights = tf.expand_dims(tf.div(top_k_xvals,x_sums_repeated), 1)
//  [python]
top_k_xvals, top_k_indices = tf.nn.top_k(tf.negative(distance), k=k)
top_k_xvals = tf.truediv(1.0, top_k_xvals)
x_sums = tf.expand_dims(tf.reduce_sum(top_k_xvals, 1),1)
x_sums_repeated = tf.matmul(x_sums,tf.ones([1, k], tf.float32))
x_val_weights = tf.expand_dims(tf.divide(top_k_xvals,x_sums_repeated), 1)
//  [python]
def sigma(x):
    return tf.div(tf.constant(1.0),
                  tf.add(tf.constant(1.0), tf.exp(tf.negative(x))))
//  [python]
def sigma(x):
    return tf.divide(tf.constant(1.0),
                  tf.add(tf.constant(1.0), tf.exp(tf.negative(x))))
//  [python]
def rescale_input(x):
  """Rescales image input to be in range [0,1]."""

  current_min = tf.reduce_min(x)
  current_max = tf.reduce_max(x)

  # we add an epsilon value to prevent division by zero
  epsilon = 1e-5
  rescaled_x = tf.div(
      tf.subtract(x, current_min),
      tf.maximum(tf.subtract(current_max, current_min), epsilon))
  return rescaled_x
//  [python]
def rescale_input(x):
  """Rescales image input to be in range [0,1]."""

  current_min = tf.reduce_min(x)
  current_max = tf.reduce_max(x)

  # we add an epsilon value to prevent division by zero
  epsilon = 1e-5
  rescaled_x = tf.divide(
      tf.subtract(x, current_min),
      tf.maximum(tf.subtract(current_max, current_min), epsilon))
  return rescaled_x
//  [python]
def _accuracy(self):
        # shape: [batch_size]
        aff = self.link_pred_layer.affinity(self.outputs1, self.outputs2)
       # shape : [batch_size x num_neg_samples]
        self.neg_aff = self.link_pred_layer.neg_cost(self.outputs1, self.neg_outputs)
        #self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, FLAGS.neg_sample_size])
        self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, input_neg_sample_size])
        _aff = tf.expand_dims(aff, axis=1)
        self.aff_all = tf.concat(axis=1, values=[self.neg_aff, _aff])
        size = tf.shape(self.aff_all)[1]
        _, indices_of_ranks = tf.nn.top_k(self.aff_all, k=size)
        _, self.ranks = tf.nn.top_k(-indices_of_ranks, k=size)
        self.mrr = tf.reduce_mean(tf.div(1.0, tf.cast(self.ranks[:, -1] + 1, tf.float32)))
        tf.summary.scalar('mrr', self.mrr)
//  [python]
def _accuracy(self):
        # shape: [batch_size]
        aff = self.link_pred_layer.affinity(self.outputs1, self.outputs2)
       # shape : [batch_size x num_neg_samples]
        self.neg_aff = self.link_pred_layer.neg_cost(self.outputs1, self.neg_outputs)
        #self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, FLAGS.neg_sample_size])
        self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, input_neg_sample_size])
        _aff = tf.expand_dims(aff, axis=1)
        self.aff_all = tf.concat(axis=1, values=[self.neg_aff, _aff])
        size = tf.shape(self.aff_all)[1]
        _, indices_of_ranks = tf.nn.top_k(self.aff_all, k=size)
        _, self.ranks = tf.nn.top_k(-indices_of_ranks, k=size)
        self.mrr = tf.reduce_mean(tf.divide(1.0, tf.cast(self.ranks[:, -1] + 1, tf.float32)))
        tf.summary.scalar('mrr', self.mrr)
//  [python]
def _accuracy(self):
        # shape: [batch_size]
        aff = self.link_pred_layer.affinity(self.outputs1, self.outputs2)
        # shape : [batch_size x num_neg_samples]
        self.neg_aff = self.link_pred_layer.neg_cost(self.outputs1, self.neg_outputs)
        #self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, FLAGS.neg_sample_size])
        self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, input_neg_sample_size])
        _aff = tf.expand_dims(aff, axis=1)
        self.aff_all = tf.concat(axis=1, values=[self.neg_aff, _aff])
        size = tf.shape(self.aff_all)[1]
        _, indices_of_ranks = tf.nn.top_k(self.aff_all, k=size)
        _, self.ranks = tf.nn.top_k(-indices_of_ranks, k=size)
        self.mrr = tf.reduce_mean(tf.div(1.0, tf.cast(self.ranks[:, -1] + 1, tf.float32)))
        tf.summary.scalar('mrr', self.mrr)
//  [python]
def _accuracy(self):
        # shape: [batch_size]
        aff = self.link_pred_layer.affinity(self.outputs1, self.outputs2)
        # shape : [batch_size x num_neg_samples]
        self.neg_aff = self.link_pred_layer.neg_cost(self.outputs1, self.neg_outputs)
        #self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, FLAGS.neg_sample_size])
        self.neg_aff = tf.reshape(self.neg_aff, [self.batch_size, input_neg_sample_size])
        _aff = tf.expand_dims(aff, axis=1)
        self.aff_all = tf.concat(axis=1, values=[self.neg_aff, _aff])
        size = tf.shape(self.aff_all)[1]
        _, indices_of_ranks = tf.nn.top_k(self.aff_all, k=size)
        _, self.ranks = tf.nn.top_k(-indices_of_ranks, k=size)
        self.mrr = tf.reduce_mean(tf.divide(1.0, tf.cast(self.ranks[:, -1] + 1, tf.float32)))
        tf.summary.scalar('mrr', self.mrr)
//  [python]
def cross_entropy_sequence_loss(logits, targets, sequence_length, max_length=None):
    """Calculates the per-example cross-entropy loss for a sequence of logits and
    masks out all losses passed the sequence length.

    Args:
        logits: Logits of shape `[T, B, vocab_size]`
        targets: Target classes of shape `[T, B]`
        sequence_length: An int32 tensor of shape `[B]` corresponding
                to the length of each input

    Returns:
        Loss: A tensor of shape [T, B] that contains the loss per example, per time step.
    """
    with tf.name_scope("cross_entropy_sequence_loss"):
        losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=logits, labels=targets)

        if max_length is None:
            max_length = targets.get_shape().as_list()[1]

        # Mask out the losses we don't care about
        loss_mask = tf.sequence_mask(sequence_length, max_length)
        losses = losses * tf.to_float(loss_mask)
        loss_per_batch = tf.div(tf.reduce_sum(losses, 1), tf.reduce_sum(tf.to_float(loss_mask), 1))
    return tf.reduce_mean(loss_per_batch)
//  [python]
def cross_entropy_sequence_loss(logits, targets, sequence_length, max_length=None):
    """Calculates the per-example cross-entropy loss for a sequence of logits and
    masks out all losses passed the sequence length.

    Args:
        logits: Logits of shape `[T, B, vocab_size]`
        targets: Target classes of shape `[T, B]`
        sequence_length: An int32 tensor of shape `[B]` corresponding
                to the length of each input

    Returns:
        Loss: A tensor of shape [T, B] that contains the loss per example, per time step.
    """
    with tf.name_scope("cross_entropy_sequence_loss"):
        losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=logits, labels=targets)

        if max_length is None:
            max_length = targets.get_shape().as_list()[1]

        # Mask out the losses we don't care about
        loss_mask = tf.sequence_mask(sequence_length, max_length)
        losses = losses * tf.to_float(loss_mask)
        loss_per_batch = tf.divide(tf.reduce_sum(losses, 1), tf.reduce_sum(tf.to_float(loss_mask), 1))
    return tf.reduce_mean(loss_per_batch)
//  [python]
def g_out_phaseless(phat,pvar,y,wvar):
    #Untested. To be used with D-prGAMP

    y_abs = y
    phat_abs = tf.abs(phat)
    B = 2.* tf.div(tf.multiply(y_abs,phat_abs),wvar+pvar)
    I1overI0 = tf.minimum(tf.div(B,tf.sqrt(tf.square(B)+4)),tf.div(B,0.5+tf.sqrt(tf.square(B)+0.25)))
    y_sca = tf.div(y_abs,1.+tf.div(wvar,pvar))
    phat_sca = tf.div(phat_abs,1.+tf.div(pvar,wvar))
    zhat = tf.multiply(tf.add(phat_sca,tf.multiply(y_sca,I1overI0)),tf.sign(phat))

    sigma2_z = tf.add(tf.add(tf.square(y_sca),tf.square(phat_sca)),tf.subtract(tf.div(1.+tf.multiply(B,I1overI0),tf.add(tf.div(1.,wvar),tf.div(1.,pvar))),tf.square(tf.abs(zhat))))

    g = tf.multiply(tf.div(1.,pvar),tf.subtract(zhat,phat))
    dg = tf.multiply(tf.div(1.,pvar),tf.subtract(tf.div(tf.reduce_mean(sigma2_z,axis=(1,2,3)),pvar),1))

    return g,dg
//  [python]
def g_out_phaseless(phat,pvar,y,wvar):
    #Untested. To be used with D-prGAMP

    y_abs = y
    phat_abs = tf.abs(phat)
    B = 2.* tf.divide(tf.multiply(y_abs,phat_abs),wvar+pvar)
    I1overI0 = tf.minimum(tf.divide(B,tf.sqrt(tf.square(B)+4)),tf.divide(B,0.5+tf.sqrt(tf.square(B)+0.25)))
    y_sca = tf.divide(y_abs,1.+tf.divide(wvar,pvar))
    phat_sca = tf.divide(phat_abs,1.+tf.divide(pvar,wvar))
    zhat = tf.multiply(tf.add(phat_sca,tf.multiply(y_sca,I1overI0)),tf.sign(phat))

    sigma2_z = tf.add(tf.add(tf.square(y_sca),tf.square(phat_sca)),tf.subtract(tf.divide(1.+tf.multiply(B,I1overI0),tf.add(tf.divide(1.,wvar),tf.divide(1.,pvar))),tf.square(tf.abs(zhat))))

    g = tf.multiply(tf.divide(1.,pvar),tf.subtract(zhat,phat))
    dg = tf.multiply(tf.divide(1.,pvar),tf.subtract(tf.divide(tf.reduce_mean(sigma2_z,axis=(1,2,3)),pvar),1))

    return g,dg
//  [python]
def _setup_sparsity(self):
    """Defines the current level of sparsity required."""
    begin_step = self._spec.sparsity_function_begin_step
    end_step = self._spec.sparsity_function_end_step
    initial_sparsity = self._spec.initial_sparsity
    exponent = self._spec.sparsity_function_exponent

    if begin_step >= end_step:
      raise ValueError(
          'Pruning must begin before it can end. begin_step=%d, end_step=%d' %
          (begin_step, end_step))

    with tf.name_scope(self._spec.name):
      # is the fraction of the total training completed
      p = tf.minimum(
          1.0,
          tf.maximum(
              0.0,
              tf.div(
                  tf.cast(self._global_step - begin_step, tf.float32),
                  end_step - begin_step)))

      # slowly introduce sparsity
      sparsity = tf.cast(
          tf.add(
              tf.multiply((initial_sparsity - self._target_sparsity),
                          tf.pow(1 - p, exponent)),
              self._target_sparsity,
              name='sparsity'), tf.float32)
    return sparsity
//  [python]
def _setup_sparsity(self):
    """Defines the current level of sparsity required."""
    begin_step = self._spec.sparsity_function_begin_step
    end_step = self._spec.sparsity_function_end_step
    initial_sparsity = self._spec.initial_sparsity
    exponent = self._spec.sparsity_function_exponent

    if begin_step >= end_step:
      raise ValueError(
          'Pruning must begin before it can end. begin_step=%d, end_step=%d' %
          (begin_step, end_step))

    with tf.name_scope(self._spec.name):
      # is the fraction of the total training completed
      p = tf.minimum(
          1.0,
          tf.maximum(
              0.0,
              tf.divide(
                  tf.cast(self._global_step - begin_step, tf.float32),
                  end_step - begin_step)))

      # slowly introduce sparsity
      sparsity = tf.cast(
          tf.add(
              tf.multiply((initial_sparsity - self._target_sparsity),
                          tf.pow(1 - p, exponent)),
              self._target_sparsity,
              name='sparsity'), tf.float32)
    return sparsity
//  [python]
def _get_sparsity(self, weight_name):
    """Return target sparsity for the given layer/weight name."""
    target_sparsity = [
        sparsity for name, sparsity in self._weight_sparsity_map.items()
        if weight_name.find(name) != -1
    ]
    if not target_sparsity:
      return self._sparsity

    if len(target_sparsity) > 1:
      raise ValueError(
          'Multiple matches in weight_sparsity_map for weight %s' % weight_name)
    return tf.mul(self._sparsity,
                  tf.div(target_sparsity[0], self._spec.target_sparsity))
//  [python]
def _get_sparsity(self, weight_name):
    """Return target sparsity for the given layer/weight name."""
    target_sparsity = [
        sparsity for name, sparsity in self._weight_sparsity_map.items()
        if weight_name.find(name) != -1
    ]
    if not target_sparsity:
      return self._sparsity

    if len(target_sparsity) > 1:
      raise ValueError(
          'Multiple matches in weight_sparsity_map for weight %s' % weight_name)
    return tf.multiply(self._sparsity,
                  tf.divide(target_sparsity[0], self._spec.target_sparsity))
//  [python]
def __call__(self, inputs, state, scope=None):
    """Run the cell with the declared dropouts."""
    orig_inputs = inputs
    if self._input_keep_prob < 1:
      # When using functional_rnn, the forward pass will be recomputed in the
      # backprop. So we need to make the dropout layer deterministic between
      # farward and backward pass. So we use stateless random to make sure the
      # generated random number is deterministic with a given seed. We also
      # want the drop out to be random across different global steps and time
      # steps. So we put both of them to the seeds.
      seeds = tf.stop_gradient(
          tf.stack([
              tf.cast(self._global_step, tf.int32) + tf.reduce_sum(state[1]),
              int(hashlib.md5(
                  inputs.name.encode("utf-8")).hexdigest()[:8], 16) & 0x7FFFFFFF
          ]))
      keep_prob = tf.convert_to_tensor(
          self._input_keep_prob, dtype=tf.float32, name="keep_prob")
      random_tensor = keep_prob + contrib_stateless.stateless_random_uniform(
          inputs.get_shape(), seed=tf.cast(seeds, tf.int32), dtype=tf.float32)
      binary_tensor = tf.cast(tf.floor(random_tensor), inputs.dtype)
      inputs = tf.div(inputs, tf.cast(keep_prob, inputs.dtype)) * binary_tensor
    with tf.variable_scope("cell_fn", reuse=tf.AUTO_REUSE):
      state0 = state[0] if self._enable else state
      output, new_state = self._cell(inputs, state0, scope=scope)
    if self._seq_len is not None and self._enable:
      seq_len = tf.reshape(self._seq_len, [-1])
      padding = tf.reshape(state[1], [-1]) < (tf.reduce_max(seq_len) - seq_len)
      output = tf.where(padding, orig_inputs, output)
      new_state = contrib_rnn.LSTMStateTuple(
          tf.where(padding, state[0].c, new_state.c),
          tf.where(padding, state[0].h, new_state.h))
    if self._enable:
      return output, [new_state, state[1] + 1]
    else:
      return output, new_state
//  [python]
def __call__(self, inputs, state, scope=None):
    """Run the cell with the declared dropouts."""
    orig_inputs = inputs
    if self._input_keep_prob < 1:
      # When using functional_rnn, the forward pass will be recomputed in the
      # backprop. So we need to make the dropout layer deterministic between
      # farward and backward pass. So we use stateless random to make sure the
      # generated random number is deterministic with a given seed. We also
      # want the drop out to be random across different global steps and time
      # steps. So we put both of them to the seeds.
      seeds = tf.stop_gradient(
          tf.stack([
              tf.cast(self._global_step, tf.int32) + tf.reduce_sum(state[1]),
              int(hashlib.md5(
                  inputs.name.encode("utf-8")).hexdigest()[:8], 16) & 0x7FFFFFFF
          ]))
      keep_prob = tf.convert_to_tensor(
          self._input_keep_prob, dtype=tf.float32, name="keep_prob")
      random_tensor = keep_prob + contrib_stateless.stateless_random_uniform(
          inputs.get_shape(), seed=tf.cast(seeds, tf.int32), dtype=tf.float32)
      binary_tensor = tf.cast(tf.floor(random_tensor), inputs.dtype)
      inputs = tf.divide(inputs, tf.cast(keep_prob, inputs.dtype)) * binary_tensor
    with tf.variable_scope("cell_fn", reuse=tf.AUTO_REUSE):
      state0 = state[0] if self._enable else state
      output, new_state = self._cell(inputs, state0, scope=scope)
    if self._seq_len is not None and self._enable:
      seq_len = tf.reshape(self._seq_len, [-1])
      padding = tf.reshape(state[1], [-1]) < (tf.reduce_max(seq_len) - seq_len)
      output = tf.where(padding, orig_inputs, output)
      new_state = contrib_rnn.LSTMStateTuple(
          tf.where(padding, state[0].c, new_state.c),
          tf.where(padding, state[0].h, new_state.h))
    if self._enable:
      return output, [new_state, state[1] + 1]
    else:
      return output, new_state
//  [python]
def approximate_split(x, num_splits, axis=0):
  """Split approximately equally into num_splits parts.

  Args:
    x: a Tensor
    num_splits: an integer
    axis: an integer.

  Returns:
    a list of num_splits Tensors.
  """
  size = shape_list(x)[axis]
  size_splits = [tf.div(size + i, num_splits) for i in range(num_splits)]
  return tf.split(x, size_splits, axis=axis)
//  [python]
def approximate_split(x, num_splits, axis=0):
  """Split approximately equally into num_splits parts.

  Args:
    x: a Tensor
    num_splits: an integer
    axis: an integer.

  Returns:
    a list of num_splits Tensors.
  """
  size = shape_list(x)[axis]
  size_splits = [tf.divide(size + i, num_splits) for i in range(num_splits)]
  return tf.split(x, size_splits, axis=axis)
//  [python]
def instance_norm(x):
  """Instance normalization layer."""
  with tf.variable_scope("instance_norm"):
    epsilon = 1e-5
    mean, var = tf.nn.moments(x, [1, 2], keep_dims=True)
    scale = tf.get_variable(
        "scale", [x.get_shape()[-1]],
        initializer=tf.truncated_normal_initializer(mean=1.0, stddev=0.02))
    offset = tf.get_variable(
        "offset", [x.get_shape()[-1]], initializer=tf.constant_initializer(0.0))
    out = scale * tf.div(x - mean, tf.sqrt(var + epsilon)) + offset

    return out
//  [python]
def instance_norm(x):
  """Instance normalization layer."""
  with tf.variable_scope("instance_norm"):
    epsilon = 1e-5
    mean, var = tf.nn.moments(x, [1, 2], keep_dims=True)
    scale = tf.get_variable(
        "scale", [x.get_shape()[-1]],
        initializer=tf.truncated_normal_initializer(mean=1.0, stddev=0.02))
    offset = tf.get_variable(
        "offset", [x.get_shape()[-1]], initializer=tf.constant_initializer(0.0))
    out = scale * tf.divide(x - mean, tf.sqrt(var + epsilon)) + offset

    return out
//  [python]
def multinomial_distribution(self, phi):
        """Probability that unit v has value l given phi: P(v=l|phi)

        Args:
            phi (tf.Tensor): linear combination of values of the previous layer
            r (float): rating scale, corresponding to the number of classes

        Returns:
            tf.Tensor:
            - A tensor of shape (r, m, Nv): This needs to be reshaped as (m, Nv, r) in the last step to allow for faster sampling when used in the multinomial function.

        """

        numerator = [
            tf.exp(tf.multiply(tf.constant(k, dtype="float32"), phi))
            for k in self.possible_ratings
        ]

        denominator = tf.reduce_sum(input_tensor=numerator, axis=0)

        prob = tf.compat.v1.div(numerator, denominator)

        return tf.transpose(a=prob, perm=[1, 2, 0])
//  [python]
def multinomial_distribution(self, phi):
        """Probability that unit v has value l given phi: P(v=l|phi)

        Args:
            phi (tf.Tensor): linear combination of values of the previous layer
            r (float): rating scale, corresponding to the number of classes

        Returns:
            tf.Tensor:
            - A tensor of shape (r, m, Nv): This needs to be reshaped as (m, Nv, r) in the last step to allow for faster sampling when used in the multinomial function.

        """

        numerator = [
            tf.exp(tf.multiply(tf.constant(k, dtype="float32"), phi))
            for k in self.possible_ratings
        ]

        denominator = tf.reduce_sum(input_tensor=numerator, axis=0)

        prob = tf.divide(numerator, denominator)

        return tf.transpose(a=prob, perm=[1, 2, 0])
//  [python]
def global_pool_1d(inputs, pooling_type="MAX", mask=None):
  """Pool elements across the last dimension.

  Useful to convert a list of vectors into a single vector so as
  to get a representation of a set.

  Args:
    inputs: A tensor of shape [batch_size, sequence_length, input_dims]
      containing the sequences of input vectors.
    pooling_type: the pooling type to use, MAX or AVR
    mask: A tensor of shape [batch_size, sequence_length] containing a
      mask for the inputs with 1's for existing elements, and 0's elsewhere.

  Returns:
    A tensor of shape [batch_size, input_dims] containing the sequences of
    transformed vectors.
  """
  with tf.name_scope("global_pool", values=[inputs]):
    if mask is not None:
      mask = tf.expand_dims(mask, axis=2)
      inputs = tf.multiply(inputs, mask)

    if pooling_type == "MAX":
      # A tf.pool can be used here, but reduce is cleaner
      output = tf.reduce_max(inputs, axis=1)
    elif pooling_type == "AVR":
      if mask is not None:
        # Some elems are dummy elems so we can't just reduce the average.
        output = tf.reduce_sum(inputs, axis=1)
        num_elems = tf.reduce_sum(mask, axis=1, keepdims=True)
        output = tf.div(output, tf.maximum(num_elems, 1))
      else:
        output = tf.reduce_mean(inputs, axis=1)

  return output
//  [python]
def global_pool_1d(inputs, pooling_type="MAX", mask=None):
  """Pool elements across the last dimension.

  Useful to convert a list of vectors into a single vector so as
  to get a representation of a set.

  Args:
    inputs: A tensor of shape [batch_size, sequence_length, input_dims]
      containing the sequences of input vectors.
    pooling_type: the pooling type to use, MAX or AVR
    mask: A tensor of shape [batch_size, sequence_length] containing a
      mask for the inputs with 1's for existing elements, and 0's elsewhere.

  Returns:
    A tensor of shape [batch_size, input_dims] containing the sequences of
    transformed vectors.
  """
  with tf.name_scope("global_pool", values=[inputs]):
    if mask is not None:
      mask = tf.expand_dims(mask, axis=2)
      inputs = tf.multiply(inputs, mask)

    if pooling_type == "MAX":
      # A tf.pool can be used here, but reduce is cleaner
      output = tf.reduce_max(inputs, axis=1)
    elif pooling_type == "AVR":
      if mask is not None:
        # Some elems are dummy elems so we can't just reduce the average.
        output = tf.reduce_sum(inputs, axis=1)
        num_elems = tf.reduce_sum(mask, axis=1, keepdims=True)
        output = tf.divide(output, tf.maximum(num_elems, 1))
      else:
        output = tf.reduce_mean(inputs, axis=1)

  return output
//  [python]
def _length_penalty(sequence_lengths, penalty_factor, dtype):
  """Calculates the length penalty. See https://arxiv.org/abs/1609.08144.

  Returns the length penalty tensor:
  ```
  [(5+sequence_lengths)/6]**penalty_factor
  ```
  where all operations are performed element-wise.

  Args:
    sequence_lengths: `Tensor`, the sequence lengths of each hypotheses.
    penalty_factor: A scalar that weights the length penalty.
    dtype: dtype of result.

  Returns:
    If the penalty is `0`, returns the scalar `1.0`.  Otherwise returns
    the length penalty factor, a tensor with the same shape as
    `sequence_lengths`.
  """
  penalty_factor = tf.convert_to_tensor(
      penalty_factor, name="penalty_factor", dtype=dtype)
  penalty_factor.set_shape(())  # penalty should be a scalar.
  static_penalty = contrib_util.constant_value(penalty_factor)
  if static_penalty is not None and static_penalty == 0:
    return 1.0
  length_penalty_const = 5.0
  return tf.div((length_penalty_const + tf.cast(sequence_lengths, dtype))
                **penalty_factor, (length_penalty_const + 1.)**penalty_factor)
//  [python]
def _length_penalty(sequence_lengths, penalty_factor, dtype):
  """Calculates the length penalty. See https://arxiv.org/abs/1609.08144.

  Returns the length penalty tensor:
  
  [(5+sequence_lengths)/6]**penalty_factor
  
  where all operations are performed element-wise.

  Args:
    sequence_lengths: `Tensor`, the sequence lengths of each hypotheses.
    penalty_factor: A scalar that weights the length penalty.
    dtype: dtype of result.

  Returns:
    If the penalty is `0`, returns the scalar `1.0`.  Otherwise returns
    the length penalty factor, a tensor with the same shape as
    `sequence_lengths`.
  """
  penalty_factor = tf.convert_to_tensor(
      penalty_factor, name="penalty_factor", dtype=dtype)
  penalty_factor.set_shape(())  # penalty should be a scalar.
  static_penalty = contrib_util.constant_value(penalty_factor)
  if static_penalty is not None and static_penalty == 0:
    return 1.0
  length_penalty_const = 5.0
  return tf.divide((length_penalty_const + tf.cast(sequence_lengths, dtype))
                **penalty_factor, (length_penalty_const + 1.)**penalty_factor)
//  [python]
def graph_cnn(inputs, object_mask, num_layers, hidden_size, dropout,
              adjacency_matrix, norm_type="layer", norm_epsilon=0.001,
              test=False):
  """Encodes a screen using Graph Convolution Networks.

  Args:
    inputs: [batch_size, num_steps, max_object_count, depth].
    object_mask: [batch_size, num_steps, max_object_count].
    num_layers: the number of layers.
    hidden_size: the hidden layer size.
    dropout: dropout ratio.
    adjacency_matrix: the adjacency matrix
        [batch_size, num_steps, max_object_count, max_object_count].
    norm_type: the norm_type.
    norm_epsilon: norm_epsilon.
    test: whether it's in the test mode.
  Returns:
    hidden: a Tensor of shape
        [batch_size, num_steps, max_object_count, depth]
  """
  # [batch_size, num_steps, max_num_objects, max_num_objects]
  normalizer = tf.div(1., tf.sqrt(tf.reduce_sum(
      adjacency_matrix, -1, keepdims=True)))
  normalizer = normalizer * tf.expand_dims(tf.expand_dims(
      tf.eye(tf.shape(normalizer)[-2]), 0), 0)
  adjacency_matrix = tf.matmul(
      tf.matmul(normalizer, adjacency_matrix), normalizer)
  hidden = inputs
  for layer in range(num_layers):
    with tf.variable_scope("gcn_layer_" + str(layer), reuse=tf.AUTO_REUSE):
      hidden = tf.matmul(adjacency_matrix, hidden)
      # [batch_size, num_steps, max_num_objects, depth]
      if not test:
        hidden = tf.layers.dense(inputs=hidden, units=hidden_size)
        hidden = common_layers.apply_norm(
            hidden, norm_type, hidden_size,
            epsilon=norm_epsilon)
        hidden = tf.nn.relu(hidden)
        hidden = tf.nn.dropout(hidden, keep_prob=1.0 - dropout)
      # zero out padding objects
      hidden = hidden * tf.expand_dims(object_mask, 3)
  return hidden
//  [python]
def graph_cnn(inputs, object_mask, num_layers, hidden_size, dropout,
              adjacency_matrix, norm_type="layer", norm_epsilon=0.001,
              test=False):
  """Encodes a screen using Graph Convolution Networks.

  Args:
    inputs: [batch_size, num_steps, max_object_count, depth].
    object_mask: [batch_size, num_steps, max_object_count].
    num_layers: the number of layers.
    hidden_size: the hidden layer size.
    dropout: dropout ratio.
    adjacency_matrix: the adjacency matrix
        [batch_size, num_steps, max_object_count, max_object_count].
    norm_type: the norm_type.
    norm_epsilon: norm_epsilon.
    test: whether it's in the test mode.
  Returns:
    hidden: a Tensor of shape
        [batch_size, num_steps, max_object_count, depth]
  """
  # [batch_size, num_steps, max_num_objects, max_num_objects]
  normalizer = tf.divide(1., tf.sqrt(tf.reduce_sum(
      adjacency_matrix, -1, keepdims=True)))
  normalizer = normalizer * tf.expand_dims(tf.expand_dims(
      tf.eye(tf.shape(normalizer)[-2]), 0), 0)
  adjacency_matrix = tf.matmul(
      tf.matmul(normalizer, adjacency_matrix), normalizer)
  hidden = inputs
  for layer in range(num_layers):
    with tf.variable_scope("gcn_layer_" + str(layer), reuse=tf.AUTO_REUSE):
      hidden = tf.matmul(adjacency_matrix, hidden)
      # [batch_size, num_steps, max_num_objects, depth]
      if not test:
        hidden = tf.layers.dense(inputs=hidden, units=hidden_size)
        hidden = common_layers.apply_norm(
            hidden, norm_type, hidden_size,
            epsilon=norm_epsilon)
        hidden = tf.nn.relu(hidden)
        hidden = tf.nn.dropout(hidden, keep_prob=1.0 - dropout)
      # zero out padding objects
      hidden = hidden * tf.expand_dims(object_mask, 3)
  return hidden
//  [python]
def reduced_by_avg(self):
    output_layer = self.output_layer
    output_layer = tf.expand_dims(self.segment_mask, axis=-1) * output_layer
    output_layer = tf.reduce_sum(output_layer, axis=1)  # [B, H]
    # be careful, cannot be divied by zero (no segments)
    output_layer = tf.div(output_layer,
                          tf.cast(tf.expand_dims(self.num_segments, -1), tf.float32))

    return output_layer
//  [python]
def reduced_by_avg(self):
    output_layer = self.output_layer
    output_layer = tf.expand_dims(self.segment_mask, axis=-1) * output_layer
    output_layer = tf.reduce_sum(output_layer, axis=1)  # [B, H]
    # be careful, cannot be divied by zero (no segments)
    output_layer = tf.divide(output_layer,
                          tf.cast(tf.expand_dims(self.num_segments, -1), tf.float32))

    return output_layer
//  [python]
def r_squared(y_true, y_pred):
    residual = tf.reduce_sum(tf.square(tf.subtract(y_true, y_pred)))
    total = tf.reduce_sum(tf.square(tf.subtract(y_true, tf.reduce_mean(y_true))))
    r2 = tf.subtract(1.0, tf.div(residual, total))
    return r2
//  [python]
def r_squared(y_true, y_pred):
    residual = tf.reduce_sum(tf.square(tf.subtract(y_true, y_pred)))
    total = tf.reduce_sum(tf.square(tf.subtract(y_true, tf.reduce_mean(y_true))))
    r2 = tf.subtract(1.0, tf.divide(residual, total))
    return r2
//  [python]
def testAssertDivideByZero(self):
    with self.test_session() as sess:
      epsilon = tf.convert_to_tensor(1e-20)
      x = tf.convert_to_tensor(0.0)
      y = tf.convert_to_tensor(1.0)
      z = tf.convert_to_tensor(2.0)
      # assert(epsilon < y)
      # z / y
      with sess.graph.control_dependencies(
          [tf.Assert(tf.less(epsilon, y), ["Divide-by-zero"])]):
        out = tf.div(z, y)
      self.assertAllEqual(2.0, out.eval())
      # assert(epsilon < x)
      # z / x
      #
      # This tests printing out multiple tensors
      with sess.graph.control_dependencies(
          [tf.Assert(tf.less(epsilon, x),
                     ["Divide-by-zero", "less than x"])]):
        out = tf.div(z, x)
      with self.assertRaisesOpError("less than x"):
        out.eval()
//  [python]
def testAssertDivideByZero(self):
    with self.test_session() as sess:
      epsilon = tf.convert_to_tensor(1e-20)
      x = tf.convert_to_tensor(0.0)
      y = tf.convert_to_tensor(1.0)
      z = tf.convert_to_tensor(2.0)
      # assert(epsilon < y)
      # z / y
      with sess.graph.control_dependencies(
          [tf.Assert(tf.less(epsilon, y), ["Divide-by-zero"])]):
        out = tf.divide(z, y)
      self.assertAllEqual(2.0, out.eval())
      # assert(epsilon < x)
      # z / x
      #
      # This tests printing out multiple tensors
      with sess.graph.control_dependencies(
          [tf.Assert(tf.less(epsilon, x),
                     ["Divide-by-zero", "less than x"])]):
        out = tf.divide(z, x)
      with self.assertRaisesOpError("less than x"):
        out.eval()
//  [python]
def w_fold(w, gama, var, epsilon):
    return tf.compat.v1.div(tf.multiply(gama, w), tf.sqrt(var + epsilon))
//  [python]
def w_fold(w, gama, var, epsilon):
    return tf.divide(tf.multiply(gama, w), tf.sqrt(var + epsilon))
//  [python]
def localization_loss(start_logits, end_logits, y1, y2, configs):

    start_prob = tf.nn.softmax(start_logits, axis=1)
    end_prob = tf.nn.softmax(end_logits, axis=1)

    outer = tf.matmul(tf.expand_dims(start_prob, axis=2),
                      tf.expand_dims(end_prob, axis=1))
    outer = tf.matrix_band_part(outer, num_lower=0, num_upper=-1)

    batch_size = tf.shape(outer)[0]
    length = tf.shape(outer)[1]
    outer_reshape = tf.reshape(outer, [batch_size, -1])
    if configs.mode == "train":
        k = 16
    elif configs.mode == "test":
        k = 16
    topk, topk_index = tf.nn.top_k(outer_reshape, k, sorted=False)
    dx = tf.div(topk_index, length)
    dy = tf.mod(topk_index, length)

    mask_dx = tf.sequence_mask(lengths=dx, maxlen=length, dtype=tf.float32)
    mask_dy = tf.sequence_mask(lengths=dy + 1, maxlen=length, dtype=tf.float32)
    mask = mask_dy - mask_dx

    starts = tf.reduce_max(outer, axis=2)
    ends = tf.reduce_max(outer, axis=1)
    start_index = tf.argmax(starts, axis=1)
    end_index = tf.argmax(ends, axis=1)

    start_losses = tf.nn.softmax_cross_entropy_with_logits_v2(
        logits=start_logits, labels=y1)
    end_losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=end_logits,
                                                            labels=y2)
    loss = tf.reduce_mean(start_losses + end_losses)

    return start_prob, end_prob, start_index, end_index, mask, dx, dy, loss
//  [python]
def localization_loss(start_logits, end_logits, y1, y2, configs):

    start_prob = tf.nn.softmax(start_logits, axis=1)
    end_prob = tf.nn.softmax(end_logits, axis=1)

    outer = tf.matmul(tf.expand_dims(start_prob, axis=2),
                      tf.expand_dims(end_prob, axis=1))
    outer = tf.matrix_band_part(outer, num_lower=0, num_upper=-1)

    batch_size = tf.shape(outer)[0]
    length = tf.shape(outer)[1]
    outer_reshape = tf.reshape(outer, [batch_size, -1])
    if configs.mode == "train":
        k = 16
    elif configs.mode == "test":
        k = 16
    topk, topk_index = tf.nn.top_k(outer_reshape, k, sorted=False)
    dx = tf.divide(topk_index, length)
    dy = tf.mod(topk_index, length)

    mask_dx = tf.sequence_mask(lengths=dx, maxlen=length, dtype=tf.float32)
    mask_dy = tf.sequence_mask(lengths=dy + 1, maxlen=length, dtype=tf.float32)
    mask = mask_dy - mask_dx

    starts = tf.reduce_max(outer, axis=2)
    ends = tf.reduce_max(outer, axis=1)
    start_index = tf.argmax(starts, axis=1)
    end_index = tf.argmax(ends, axis=1)

    start_losses = tf.nn.softmax_cross_entropy_with_logits_v2(
        logits=start_logits, labels=y1)
    end_losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=end_logits,
                                                            labels=y2)
    loss = tf.reduce_mean(start_losses + end_losses)

    return start_prob, end_prob, start_index, end_index, mask, dx, dy, loss
//  [python]
def bias_fold(beta, gama, mean, var, epsilon):
    return tf.subtract(beta, tf.compat.v1.div(tf.multiply(gama, mean), tf.sqrt(var + epsilon)))
//  [python]
def bias_fold(beta, gama, mean, var, epsilon):
    return tf.subtract(beta, tf.divide(tf.multiply(gama, mean), tf.sqrt(var + epsilon)))
//  [python]
def visit_div(self, op: d5.ops.Div, network: TensorflowNetwork):
        A, B = network.fetch_internal_tensors([op.i_A, op.i_B])
        C = tf.div(A, B)
        network.feed_internal_tensor(op.o_C, C)
//  [python]
def visit_div(self, op: d5.ops.Div, network: TensorflowNetwork):
        A, B = network.fetch_internal_tensors([op.i_A, op.i_B])
        C = tf.divide(A, B)
        network.feed_internal_tensor(op.o_C, C)
//  [python]
def create_loss_function(self, is_regression):
        with tf.compat.v1.variable_scope('loss') as scope:
            self.print_ext('Creating loss function and summaries')
            if is_regression:
                prediction = tf.squeeze(tf.cast(self.net.current_V, tf.float32))
                labels = tf.squeeze(self.net.labels)
                
                mse = tf.losses.mean_squared_error(labels, prediction, scope='loss')
                
                total_error = tf.reduce_sum(tf.square(tf.subtract(labels, tf.reduce_mean(labels))))
                unexplained_error = tf.reduce_sum(tf.square(tf.subtract(labels, prediction)))
                R_squared = tf.subtract(float(1), tf.div(unexplained_error, total_error))
                
                tf.add_to_collection('losses', mse)
                tf.summary.scalar('loss', mse)

                self.max_r2_train = tf.Variable(tf.zeros([]), name="max_r2_train")
                self.max_r2_test = tf.Variable(tf.zeros([]), name="max_r2_test")
                
                max_r2 = tf.cond(self.net.is_training, lambda: tf.assign(self.max_r2_train, tf.maximum(self.max_r2_train, R_squared)), lambda: tf.assign(self.max_r2_test, tf.maximum(self.max_r2_test, R_squared)))
                tf.summary.scalar('max_r2', max_r2)
                tf.summary.scalar('accuracy', R_squared)

                self.reports['r2'] = R_squared
                self.reports['max r2'] = max_r2
                self.reports['mse'] = mse
            else:

                cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.net.current_V, labels=self.net.labels))

                correct_prediction = tf.cast(tf.equal(tf.argmax(self.net.current_V, 1), self.net.labels), tf.float32)
                accuracy = tf.reduce_mean(correct_prediction)

                # we have 2 variables that will keep track of the best accuracy obtained in training/testing batch
                # SHOULD ONLY BE USED IF test_batch_size == ALL TEST SAMPLES
                
                self.max_acc_train = tf.Variable(tf.zeros([]), name="max_acc_train")
                self.max_acc_test = tf.Variable(tf.zeros([]), name="max_acc_test")
                max_acc = tf.cond(self.net.is_training, lambda: tf.assign(self.max_acc_train, tf.maximum(self.max_acc_train, accuracy)), lambda: tf.assign(self.max_acc_test, tf.maximum(self.max_acc_test, accuracy)))

                tf.add_to_collection('losses', cross_entropy)
                tf.summary.scalar('accuracy', accuracy)
                tf.summary.scalar('max_accuracy', max_acc)
                tf.summary.scalar('cross_entropy', cross_entropy)
                
                self.reports['accuracy'] = accuracy
                self.reports['max acc.'] = max_acc
                self.reports['cross_entropy'] = cross_entropy