jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def ae_loss_for_single_img(self,net,indexs,nr):
        indexs = indexs[:nr]
        fnr = tf.cast(nr,tf.float32)
        with tf.name_scope("pull_loss"):
            net = tf.reshape(net,[-1])
            old_shape = btf.combined_static_and_dynamic_shape(indexs)
            ids = tf.gather(net,tf.nn.relu(tf.reshape(indexs,[-1])))
            ids = tf.reshape(ids,old_shape)
            weights = tf.cast(tf.greater_equal(indexs,0),tf.float32)
            mean = tf.reduce_sum(ids*weights,axis=-1,keepdims=True)/tf.maximum(tf.reduce_sum(weights,axis=-1,keepdims=True),1e-8)
            pull_loss = tf.square(ids-mean)*weights
            pull_loss = tf.reduce_sum(pull_loss)/fnr

        with tf.name_scope("push_loss"):
            range = tf.range(nr)
            X,Y = tf.meshgrid(range,range)
            X = tf.reshape(X,[-1])
            Y = tf.reshape(Y,[-1])
            mask = tf.not_equal(X,Y)
            ftotal_nr = tf.reduce_sum(tf.cast(mask,tf.float32))+1e-8
            X = tf.boolean_mask(X,mask)
            Y = tf.boolean_mask(Y,mask)
            loss = -tf.square(tf.gather(mean,X)-tf.gather(mean,Y))
            loss = tf.exp(loss)
            loss = tf.reduce_sum(loss)
            loss = loss/ftotal_nr
            push_loss = loss*0.5

        return pull_loss,push_loss
//  [python]
def sample(self, sample_shape=(), seed=None):
    # Straight through biased gradient estimator.
    sample = t.cast(super().sample(sample_shape, seed), self._sample_dtype)
    probs = super().probs_parameter()
    while len(probs.shape) < len(sample.shape):
      probs = probs[None]
    sample += t.cast(probs - tf.stop_gradient(probs), self._sample_dtype)
    return sample
//  [python]
def sample(self, sample_shape=(), seed=None):
    # Straight through biased gradient estimator.
    sample = tf.cast(super().sample(sample_shape, seed), self._sample_dtype)
    probs = super().probs_parameter()
    while len(probs.shape) < len(sample.shape):
      probs = probs[None]
    sample += tf.cast(probs - tf.stop_gradient(probs), self._sample_dtype)
    return sample
//  [python]
def sample(self, sample_shape=(), seed=None):
    return t.cast(super().sample(sample_shape, seed), self._sample_dtype)
//  [python]
def sample(self, sample_shape=(), seed=None):
    return tf.cast(super().sample(sample_shape, seed), self._sample_dtype)
//  [python]
def __init__(self, loc, scale, threshold=1, **kwargs):
    self._threshold = t.cast(threshold, loc.dtype)
    super().__init__(loc, scale, **kwargs)
//  [python]
def __init__(self, loc, scale, threshold=1, **kwargs):
    self._threshold = tf.cast(threshold, loc.dtype)
    super().__init__(loc, scale, **kwargs)
//  [python]
def adjust(self,ans,det):
        locs = ans[...,:2]
        values = ans[...,2]
        x,y = tf.unstack(locs,axis=-1)
        org_x,org_y = x,y
        xx = t.cast(x,tf.int32)
        yy = t.cast(y,tf.int32)
        B,H,W,num_keypoints = btf.combined_static_and_dynamic_shape(det)
        det = tf.transpose(det,[0,3,1,2])
        det = tf.reshape(det,[B*num_keypoints,H*W])
        yy_p = tf.minimum(yy+1,H-1)
        yy_n = tf.maximum(yy-1,0)
        xx_p = tf.minimum(xx+1,W-1)
        xx_n = tf.maximum(xx-1,0)

        def get_values(_xx,_yy):
            B,N,KN = btf.combined_static_and_dynamic_shape(_xx)
            _xx = tf.transpose(_xx,[0,2,1])
            _yy = tf.transpose(_yy,[0,2,1])
            _xx = tf.reshape(_xx,[B*KN,N])
            _yy = tf.reshape(_yy,[B*KN,N])
            index = _xx+_yy*W
            vals = tf.batch_gather(det,index)
            vals = tf.reshape(vals,[B,KN,N])
            vals = tf.transpose(vals,[0,2,1])
            return vals

        y_p = y+0.25
        y_n = y-0.25
        x_p = x+0.25
        x_n = x-0.25

        y = tf.where(get_values(xx,yy_p)>get_values(xx,yy_n),y_p,y_n)
        x = tf.where(get_values(xx_p,yy)>get_values(xx_n,yy),x_p,x_n)

        x = x+0.5
        y = y+0.5

        x = tf.where(values>0,x,org_x)
        y = tf.where(values>0,y,org_y)

        loc = tf.stack([x,y],axis=-1)

        B,N,KP,C = btf.combined_static_and_dynamic_shape(ans)
        _,data = tf.split(ans,[2,C-2],axis=-1)

        return tf.concat([loc,data],axis=-1)
//  [python]
def adjust(self,ans,det):
        locs = ans[...,:2]
        values = ans[...,2]
        x,y = tf.unstack(locs,axis=-1)
        org_x,org_y = x,y
        xx = tf.cast(x,tf.int32)
        yy = tf.cast(y,tf.int32)
        B,H,W,num_keypoints = btf.combined_static_and_dynamic_shape(det)
        det = tf.transpose(det,[0,3,1,2])
        det = tf.reshape(det,[B*num_keypoints,H*W])
        yy_p = tf.minimum(yy+1,H-1)
        yy_n = tf.maximum(yy-1,0)
        xx_p = tf.minimum(xx+1,W-1)
        xx_n = tf.maximum(xx-1,0)

        def get_values(_xx,_yy):
            B,N,KN = btf.combined_static_and_dynamic_shape(_xx)
            _xx = tf.transpose(_xx,[0,2,1])
            _yy = tf.transpose(_yy,[0,2,1])
            _xx = tf.reshape(_xx,[B*KN,N])
            _yy = tf.reshape(_yy,[B*KN,N])
            index = _xx+_yy*W
            vals = tf.batch_gather(det,index)
            vals = tf.reshape(vals,[B,KN,N])
            vals = tf.transpose(vals,[0,2,1])
            return vals

        y_p = y+0.25
        y_n = y-0.25
        x_p = x+0.25
        x_n = x-0.25

        y = tf.where(get_values(xx,yy_p)>get_values(xx,yy_n),y_p,y_n)
        x = tf.where(get_values(xx_p,yy)>get_values(xx_n,yy),x_p,x_n)

        x = x+0.5
        y = y+0.5

        x = tf.where(values>0,x,org_x)
        y = tf.where(values>0,y,org_y)

        loc = tf.stack([x,y],axis=-1)

        B,N,KP,C = btf.combined_static_and_dynamic_shape(ans)
        _,data = tf.split(ans,[2,C-2],axis=-1)

        return tf.concat([loc,data],axis=-1)
//  [python]
def _build(self):
        mels = tf.random.uniform(shape=[2, 20, 80], dtype=tf.float32)
        self(mels, training=tf.convert_to_tensor(True, dtype=tf.bool))
//  [python]
def _build(self):
        mels = tf.random.uniform(shape=[2, 20, 80], dtype=tf.float32)
        self(mels, training=tf.cast(True, tf.bool))
//  [python]
def schedule(string, step):
  try:
    return float(string)
  except ValueError:
    step = t.cast(step, t.float32)
    match = re.match(r'linear\((.+),(.+),(.+)\)', string)
    if match:
      initial, final, duration = [float(group) for group in match.groups()]
      mix = tf.clip_by_value(step / duration, 0, 1)
      return (1 - mix) * initial + mix * final
    match = re.match(r'warmup\((.+),(.+)\)', string)
    if match:
      warmup, value = [float(group) for group in match.groups()]
      scale = tf.clip_by_value(step / warmup, 0, 1)
      return scale * value
    match = re.match(r'exp\((.+),(.+),(.+)\)', string)
    if match:
      initial, final, halflife = [float(group) for group in match.groups()]
      return (initial - final) * 0.5 ** (step / halflife) + final
    raise NotImplementedError(string)
//  [python]
def schedule(string, step):
  try:
    return float(string)
  except ValueError:
    step = tf.cast(step, tf.float32)
    match = re.match(r'linear\((.+),(.+),(.+)\)', string)
    if match:
      initial, final, duration = [float(group) for group in match.groups()]
      mix = tf.clip_by_value(step / duration, 0, 1)
      return (1 - mix) * initial + mix * final
    match = re.match(r'warmup\((.+),(.+)\)', string)
    if match:
      warmup, value = [float(group) for group in match.groups()]
      scale = tf.clip_by_value(step / warmup, 0, 1)
      return scale * value
    match = re.match(r'exp\((.+),(.+),(.+)\)', string)
    if match:
      initial, final, halflife = [float(group) for group in match.groups()]
      return (initial - final) * 0.5 ** (step / halflife) + final
    raise NotImplementedError(string)
//  [python]
def loss_func(targets, logits):
    mask = tf.math.logical_not(tf.math.equal(targets, 0))
    mask = tf.to_int64(mask)
    loss = crossentropy(targets, logits, sample_weight=mask)

    return loss
//  [python]
def loss_func(targets, logits):
    mask = tf.math.logical_not(tf.math.equal(targets, 0))
    mask = tf.cast(mask, dtype=tf.int64)
    loss = crossentropy(targets, logits, sample_weight=mask)

    return loss
//  [python]
def __init__(self,
                 enc_mat,
                 is_pcm=False,
                 dtype=tf.float32,
                 **kwargs):

        super().__init__(dtype=dtype, **kwargs)

        # tf.int8 currently not supported by tf.matmult
        assert (dtype in
               (tf.float16, tf.float32, tf.float64, tf.int32, tf.int64)), \
               "Unsupported dtype."

        # check input values for consistency
        assert isinstance(is_pcm, bool), \
                                    'is_parity_check must be bool.'

        # verify that enc_mat is binary
        assert ((enc_mat==0) | (enc_mat==1)).all(), "enc_mat is not binary."
        assert (len(enc_mat.shape)==2), "enc_mat must be 2-D array."

        # in case parity-check matrix is provided, convert to generator matrix
        if is_pcm:
            self._gm = pcm2gm(enc_mat, verify_results=True)
        else:
            self._gm = enc_mat

        self._k = self._gm.shape[0]
        self._n = self._gm.shape[1]
        self._coderate = self._k / self._n

        assert (self._k<=self._n), "Invalid matrix dimensions."

        self._gm = tf.cast(self._gm, dtype=self.dtype)
//  [python]
def call(self, sequence, training=True, encoder_mask=None):
        """ Forward pass for the Encoder

        Args:
            sequence: source input sequences
            training: whether training or not (for Dropout)
            encoder_mask: padding mask for the Encoder's Multi-Head Attention
        
        Returns:
            The output of the Encoder (batch_size, length, model_size)
            The alignment (attention) vectors for all layers
        """
        embed_out = self.embedding(sequence)

        embed_out *= tf.sqrt(tf.cast(self.model_size, tf.float32))
        embed_out += pes[:sequence.shape[1], :]
        embed_out = self.embedding_dropout(embed_out)

        sub_in = embed_out
        alignments = []

        for i in range(self.num_layers):
            sub_out, alignment = self.attention[i](sub_in, sub_in, encoder_mask)
            sub_out = self.attention_dropout[i](sub_out, training=training)
            sub_out = sub_in + sub_out
            sub_out = self.attention_norm[i](sub_out)
            
            alignments.append(alignment)
            ffn_in = sub_out

            ffn_out = self.dense_2[i](self.dense_1[i](ffn_in))
            ffn_out = self.ffn_dropout[i](ffn_out, training=training)
            ffn_out = ffn_in + ffn_out
            ffn_out = self.ffn_norm[i](ffn_out)

            sub_in = ffn_out

        return ffn_out, alignments
//  [python]
def call(self, sequence, training=True, encoder_mask=None):
        """ Forward pass for the Encoder

        Args:
            sequence: source input sequences
            training: whether training or not (for Dropout)
            encoder_mask: padding mask for the Encoder's Multi-Head Attention
        
        Returns:
            The output of the Encoder (batch_size, length, model_size)
            The alignment (attention) vectors for all layers
        """
        embed_out = self.embedding(sequence)

        embed_out *= tf.math.sqrt(tf.cast(self.model_size, tf.float32))
        embed_out += pes[:sequence.shape[1], :]
        embed_out = self.embedding_dropout(embed_out)

        sub_in = embed_out
        alignments = []

        for i in range(self.num_layers):
            sub_out, alignment = self.attention[i](sub_in, sub_in, encoder_mask)
            sub_out = self.attention_dropout[i](sub_out, training=training)
            sub_out = sub_in + sub_out
            sub_out = self.attention_norm[i](sub_out)
            
            alignments.append(alignment)
            ffn_in = sub_out

            ffn_out = self.dense_2[i](self.dense_1[i](ffn_in))
            ffn_out = self.ffn_dropout[i](ffn_out, training=training)
            ffn_out = ffn_in + ffn_out
            ffn_out = self.ffn_norm[i](ffn_out)

            sub_in = ffn_out

        return ffn_out, alignments
//  [python]
def grad_planner(
    state, num_actions, horizon, proposals, iterations, imagine, objective,
    kl_scale, step_size):
  dtype = prec.global_policy().compute_dtype
  B, P = list(state.values())[0].shape[0], proposals
  H, A = horizon, num_actions
  flat_state = {k: tf.repeat(v, P, 0) for k, v in state.items()}
  mean = tf.zeros((B, H, A), dtype)
  rawstd = 0.54 * tf.ones((B, H, A), dtype)
  for _ in range(iterations):
    proposals = tf.random.normal((B, P, H, A), dtype=dtype)
    with tf.GradientTape(watch_accessed_variables=False) as tape:
      tape.watch(mean)
      tape.watch(rawstd)
      std = tf.nn.softplus(rawstd)
      proposals = proposals * std[:, None] + mean[:, None]
      proposals = (
          tf.stop_gradient(tf.clip_by_value(proposals, -1, 1)) +
          proposals - tf.stop_gradient(proposals))
      flat_proposals = tf.reshape(proposals, (B * P, H, A))
      states = imagine(flat_proposals, flat_state)
      scores = objective(states)
      scores = tf.reshape(tf.reduce_sum(scores, -1), (B, P))
      div = tfd.kl_divergence(
          tfd.Normal(mean, std),
          tfd.Normal(tf.zeros_like(mean), tf.ones_like(std)))
      elbo = tf.reduce_sum(scores) - kl_scale * div
      elbo /= t.cast(tf.reduce_prod(tf.shape(scores)), dtype)
    grad_mean, grad_rawstd = tape.gradient(elbo, [mean, rawstd])
    e, v = tf.nn.moments(grad_mean, [1, 2], keepdims=True)
    grad_mean /= tf.sqrt(e * e + v + 1e-4)
    e, v = tf.nn.moments(grad_rawstd, [1, 2], keepdims=True)
    grad_rawstd /= tf.sqrt(e * e + v + 1e-4)
    mean = tf.clip_by_value(mean + step_size * grad_mean, -1, 1)
    rawstd = rawstd + step_size * grad_rawstd
  return mean[:, 0, :]
//  [python]
def grad_planner(
    state, num_actions, horizon, proposals, iterations, imagine, objective,
    kl_scale, step_size):
  dtype = prec.global_policy().compute_dtype
  B, P = list(state.values())[0].shape[0], proposals
  H, A = horizon, num_actions
  flat_state = {k: tf.repeat(v, P, 0) for k, v in state.items()}
  mean = tf.zeros((B, H, A), dtype)
  rawstd = 0.54 * tf.ones((B, H, A), dtype)
  for _ in range(iterations):
    proposals = tf.random.normal((B, P, H, A), dtype=dtype)
    with tf.GradientTape(watch_accessed_variables=False) as tape:
      tape.watch(mean)
      tape.watch(rawstd)
      std = tf.nn.softplus(rawstd)
      proposals = proposals * std[:, None] + mean[:, None]
      proposals = (
          tf.stop_gradient(tf.clip_by_value(proposals, -1, 1)) +
          proposals - tf.stop_gradient(proposals))
      flat_proposals = tf.reshape(proposals, (B * P, H, A))
      states = imagine(flat_proposals, flat_state)
      scores = objective(states)
      scores = tf.reshape(tf.reduce_sum(scores, -1), (B, P))
      div = tfd.kl_divergence(
          tfd.Normal(mean, std),
          tfd.Normal(tf.zeros_like(mean), tf.ones_like(std)))
      elbo = tf.reduce_sum(scores) - kl_scale * div
      elbo /= tf.cast(tf.reduce_prod(tf.shape(scores)), dtype)
    grad_mean, grad_rawstd = tape.gradient(elbo, [mean, rawstd])
    e, v = tf.nn.moments(grad_mean, [1, 2], keepdims=True)
    grad_mean /= tf.sqrt(e * e + v + 1e-4)
    e, v = tf.nn.moments(grad_rawstd, [1, 2], keepdims=True)
    grad_rawstd /= tf.sqrt(e * e + v + 1e-4)
    mean = tf.clip_by_value(mean + step_size * grad_mean, -1, 1)
    rawstd = rawstd + step_size * grad_rawstd
  return mean[:, 0, :]
//  [python]
def __init__(self, model_size, warmup_steps=4000):
        super(WarmupThenDecaySchedule, self).__init__()

        self.model_size = model_size
        self.model_size = tf.cast(self.model_size, tf.float32)

        self.warmup_steps = warmup_steps
//  [python]
def train_step(source_seq, target_seq_in, target_seq_out):
    """ Execute one training step (forward pass + backward pass)

    Args:
        source_seq: source sequences
        target_seq_in: input target sequences (<start> + ...)
        target_seq_out: output target sequences (... + <end>)
    
    Returns:
        The loss value of the current pass
    """
    with tf.GradientTape() as tape:
        encoder_mask = 1 - tf.to_float(tf.equal(source_seq, 0))
        # encoder_mask has shape (batch_size, source_len)
        # we need to add two more dimensions in between
        # to make it broadcastable when computing attention heads
        encoder_mask = tf.expand_dims(encoder_mask, axis=1)
        encoder_mask = tf.expand_dims(encoder_mask, axis=1)
        encoder_output, _ = encoder(source_seq, encoder_mask=encoder_mask)

        decoder_output, _, _ = decoder(
            target_seq_in, encoder_output, encoder_mask=encoder_mask)

        loss = loss_func(target_seq_out, decoder_output)

    variables = encoder.trainable_variables + decoder.trainable_variables
    gradients = tape.gradient(loss, variables)
    optimizer.apply_gradients(zip(gradients, variables))

    return loss
//  [python]
def train_step(source_seq, target_seq_in, target_seq_out):
    """ Execute one training step (forward pass + backward pass)

    Args:
        source_seq: source sequences
        target_seq_in: input target sequences (<start> + ...)
        target_seq_out: output target sequences (... + <end>)
    
    Returns:
        The loss value of the current pass
    """
    with tf.GradientTape() as tape:
        encoder_mask = 1 - tf.cast(tf.equal(source_seq, 0), dtype=tf.float32)
        # encoder_mask has shape (batch_size, source_len)
        # we need to add two more dimensions in between
        # to make it broadcastable when computing attention heads
        encoder_mask = tf.expand_dims(encoder_mask, axis=1)
        encoder_mask = tf.expand_dims(encoder_mask, axis=1)
        encoder_output, _ = encoder(source_seq, encoder_mask=encoder_mask)

        decoder_output, _, _ = decoder(
            target_seq_in, encoder_output, encoder_mask=encoder_mask)

        loss = loss_func(target_seq_out, decoder_output)

    variables = encoder.trainable_variables + decoder.trainable_variables
    gradients = tape.gradient(loss, variables)
    optimizer.apply_gradients(zip(gradients, variables))

    return loss
//  [python]
def build_inputs(self,
                   params,
                   input_context: Optional[tf.distribute.InputContext] = None):
    num_classes = self.task_config.num_classes
    input_size = self.task_config.input_size
    image_field_key = self.task_config.train_data.image_field_key
    label_field_key = self.task_config.train_data.label_field_key

    decoder = tfds_factory.get_classification_decoder(params.tfds_name)
    parser = classification_input.Parser(
        output_size=input_size[:2],
        num_classes=num_classes,
        image_field_key=image_field_key,
        label_field_key=label_field_key,
        decode_jpeg_only=params.decode_jpeg_only,
        aug_rand_hflip=params.aug_rand_hflip,
        aug_type=params.aug_type,
        color_jitter=params.color_jitter,
        random_erasing=params.random_erasing,
        dtype=params.dtype,
        crop_area_range=params.crop_area_range)

    def patch_and_mask(images, labels):
      del labels
      patches = vit.to_patch(
          images, self.task_config.patch_h, self.task_config.patch_w)
      batch_size, num_h_patches, num_w_patches = tf_utils.get_shape_list(
          patches)[:3]
      num_patches = num_h_patches * num_w_patches
      num_masked = tf.to_int32(
          self.task_config.masking_ratio * num_patches)
      r = tf.random.uniform((batch_size, num_patches))
      rand_indices = tf.argsort(r)

      masked_indices = rand_indices[:, :num_masked]
      unmasked_indices = rand_indices[:, num_masked:]
      patches_1d = tf.reshape(patches, (batch_size, num_patches, -1))
      masked_patches = tf.gather(patches_1d, masked_indices, batch_dims=1)

      if self.task_config.norm_target:
        mean = tf.reduce_mean(masked_patches, axis=-1, keepdims=True)
        var = tf.math.reduce_variance(masked_patches, axis=-1, keepdims=True)
        std = (var + 1.e-6)**.5
        masked_patches = (masked_patches - mean) / std

      return {'patches': patches,
              'masked_indices': masked_indices,
              'unmasked_indices': unmasked_indices}, masked_patches

    reader = input_reader.InputReader(
        params=params,
        decoder_fn=decoder.decode,
        parser_fn=parser.parse_fn(params.is_training),
        postprocess_fn=patch_and_mask)

    dataset = reader.read(input_context=input_context)
    return dataset
//  [python]
def build_inputs(self,
                   params,
                   input_context: Optional[tf.distribute.InputContext] = None):
    num_classes = self.task_config.num_classes
    input_size = self.task_config.input_size
    image_field_key = self.task_config.train_data.image_field_key
    label_field_key = self.task_config.train_data.label_field_key

    decoder = tfds_factory.get_classification_decoder(params.tfds_name)
    parser = classification_input.Parser(
        output_size=input_size[:2],
        num_classes=num_classes,
        image_field_key=image_field_key,
        label_field_key=label_field_key,
        decode_jpeg_only=params.decode_jpeg_only,
        aug_rand_hflip=params.aug_rand_hflip,
        aug_type=params.aug_type,
        color_jitter=params.color_jitter,
        random_erasing=params.random_erasing,
        dtype=params.dtype,
        crop_area_range=params.crop_area_range)

    def patch_and_mask(images, labels):
      del labels
      patches = vit.to_patch(
          images, self.task_config.patch_h, self.task_config.patch_w)
      batch_size, num_h_patches, num_w_patches = tf_utils.get_shape_list(
          patches)[:3]
      num_patches = num_h_patches * num_w_patches
      num_masked = tf.cast(
          self.task_config.masking_ratio * num_patches, dtype=tf.int32)
      r = tf.random.uniform((batch_size, num_patches))
      rand_indices = tf.argsort(r)

      masked_indices = rand_indices[:, :num_masked]
      unmasked_indices = rand_indices[:, num_masked:]
      patches_1d = tf.reshape(patches, (batch_size, num_patches, -1))
      masked_patches = tf.gather(patches_1d, masked_indices, batch_dims=1)

      if self.task_config.norm_target:
        mean = tf.reduce_mean(masked_patches, axis=-1, keepdims=True)
        var = tf.math.reduce_variance(masked_patches, axis=-1, keepdims=True)
        std = (var + 1.e-6)**.5
        masked_patches = (masked_patches - mean) / std

      return {'patches': patches,
              'masked_indices': masked_indices,
              'unmasked_indices': unmasked_indices}, masked_patches

    reader = input_reader.InputReader(
        params=params,
        decoder_fn=decoder.decode,
        parser_fn=parser.parse_fn(params.is_training),
        postprocess_fn=patch_and_mask)

    dataset = reader.read(input_context=input_context)
    return dataset
//  [python]
def test_step(self, data):
        x, y, sample_weight = self.unpack_training_data(data)
        (potentials, sequence_length, kernel), decode_sequence, *_ = self(
            x, training=False, return_crf_internal=True
        )

        mask_value = 0
        special_mask = tf.not_equal(y, mask_value)
        special_mask = tf.to_float(special_mask)
        potentials = tf.multiply(potentials, tf.expand_dims(special_mask, -1))

        crf_loss = self.compute_crf_loss(
            potentials, sequence_length, kernel, y, sample_weight
        )
        loss = crf_loss + tf.reduce_sum(self.losses)
        # Update metrics (includes the metric that tracks the loss)
        self.compiled_metrics.update_state(y, decode_sequence)
        # Return a dict mapping metric names to current value
        results = {m.name: m.result() for m in self.metrics}
        #results.update({"loss": loss, "crf_loss": crf_loss})  # append loss
        results.update({"crf_loss": crf_loss})  # append loss
        return results
//  [python]
def test_step(self, data):
        x, y, sample_weight = self.unpack_training_data(data)
        (potentials, sequence_length, kernel), decode_sequence, *_ = self(
            x, training=False, return_crf_internal=True
        )

        mask_value = 0
        special_mask = tf.not_equal(y, mask_value)
        special_mask = tf.cast(special_mask, tf.float32)
        potentials = tf.multiply(potentials, tf.expand_dims(special_mask, -1))

        crf_loss = self.compute_crf_loss(
            potentials, sequence_length, kernel, y, sample_weight
        )
        loss = crf_loss + tf.reduce_sum(self.losses)
        # Update metrics (includes the metric that tracks the loss)
        self.compiled_metrics.update_state(y, decode_sequence)
        # Return a dict mapping metric names to current value
        results = {m.name: m.result() for m in self.metrics}
        #results.update({"loss": loss, "crf_loss": crf_loss})  # append loss
        results.update({"crf_loss": crf_loss})  # append loss
        return results
//  [python]
def call(self, sequence, encoder_output, training=True, encoder_mask=None):
        """ Forward pass for the Decoder

        Args:
            sequence: source input sequences
            encoder_output: output of the Encoder (for computing middle attention)
            training: whether training or not (for Dropout)
            encoder_mask: padding mask for the Encoder's Multi-Head Attention
        
        Returns:
            The output of the Encoder (batch_size, length, model_size)
            The bottom alignment (attention) vectors for all layers
            The middle alignment (attention) vectors for all layers
        """
        # EMBEDDING AND POSITIONAL EMBEDDING
        embed_out = self.embedding(sequence)

        embed_out *= tf.sqrt(tf.cast(self.model_size, tf.float32))
        embed_out += pes[:sequence.shape[1], :]
        embed_out = self.embedding_dropout(embed_out)

        bot_sub_in = embed_out
        bot_alignments = []
        mid_alignments = []

        for i in range(self.num_layers):
            # BOTTOM MULTIHEAD SUB LAYER
            seq_len = bot_sub_in.shape[1]

            if training:
                mask = tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
            else:
                mask = None
            bot_sub_out, bot_alignment = self.attention_bot[i](bot_sub_in, bot_sub_in, mask)
            bot_sub_out = self.attention_bot_dropout[i](bot_sub_out, training=training)
            bot_sub_out = bot_sub_in + bot_sub_out
            bot_sub_out = self.attention_bot_norm[i](bot_sub_out)
            
            bot_alignments.append(bot_alignment)

            # MIDDLE MULTIHEAD SUB LAYER
            mid_sub_in = bot_sub_out

            mid_sub_out, mid_alignment = self.attention_mid[i](
                mid_sub_in, encoder_output, encoder_mask)
            mid_sub_out = self.attention_mid_dropout[i](mid_sub_out, training=training)
            mid_sub_out = mid_sub_out + mid_sub_in
            mid_sub_out = self.attention_mid_norm[i](mid_sub_out)
            
            mid_alignments.append(mid_alignment)

            # FFN
            ffn_in = mid_sub_out

            ffn_out = self.dense_2[i](self.dense_1[i](ffn_in))
            ffn_out = self.ffn_dropout[i](ffn_out, training=training)
            ffn_out = ffn_out + ffn_in
            ffn_out = self.ffn_norm[i](ffn_out)

            bot_sub_in = ffn_out

        logits = self.dense(ffn_out)

        return logits, bot_alignments, mid_alignments
//  [python]
def call(self, sequence, encoder_output, training=True, encoder_mask=None):
        """ Forward pass for the Decoder

        Args:
            sequence: source input sequences
            encoder_output: output of the Encoder (for computing middle attention)
            training: whether training or not (for Dropout)
            encoder_mask: padding mask for the Encoder's Multi-Head Attention
        
        Returns:
            The output of the Encoder (batch_size, length, model_size)
            The bottom alignment (attention) vectors for all layers
            The middle alignment (attention) vectors for all layers
        """
        # EMBEDDING AND POSITIONAL EMBEDDING
        embed_out = self.embedding(sequence)

        embed_out *= tf.math.sqrt(tf.cast(self.model_size, tf.float32))
        embed_out += pes[:sequence.shape[1], :]
        embed_out = self.embedding_dropout(embed_out)

        bot_sub_in = embed_out
        bot_alignments = []
        mid_alignments = []

        for i in range(self.num_layers):
            # BOTTOM MULTIHEAD SUB LAYER
            seq_len = bot_sub_in.shape[1]

            if training:
                mask = tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)
            else:
                mask = None
            bot_sub_out, bot_alignment = self.attention_bot[i](bot_sub_in, bot_sub_in, mask)
            bot_sub_out = self.attention_bot_dropout[i](bot_sub_out, training=training)
            bot_sub_out = bot_sub_in + bot_sub_out
            bot_sub_out = self.attention_bot_norm[i](bot_sub_out)
            
            bot_alignments.append(bot_alignment)

            # MIDDLE MULTIHEAD SUB LAYER
            mid_sub_in = bot_sub_out

            mid_sub_out, mid_alignment = self.attention_mid[i](
                mid_sub_in, encoder_output, encoder_mask)
            mid_sub_out = self.attention_mid_dropout[i](mid_sub_out, training=training)
            mid_sub_out = mid_sub_out + mid_sub_in
            mid_sub_out = self.attention_mid_norm[i](mid_sub_out)
            
            mid_alignments.append(mid_alignment)

            # FFN
            ffn_in = mid_sub_out

            ffn_out = self.dense_2[i](self.dense_1[i](ffn_in))
            ffn_out = self.ffn_dropout[i](ffn_out, training=training)
            ffn_out = ffn_out + ffn_in
            ffn_out = self.ffn_norm[i](ffn_out)

            bot_sub_in = ffn_out

        logits = self.dense(ffn_out)

        return logits, bot_alignments, mid_alignments
//  [python]
def _incidence_matrix_single(triangular_adjacency, *, num_edges):
    """
    Creates the corresponding incidence matrix for a graph with a particular
    adjacency matrix.

    :param triangular_adjacency: The binary adjacency matrix. Should have shape
        (n_nodes, n_nodes), and be triangular.
    :param num_edges: The number of edges to use in the output. Should be large
        enough to accommodate all the edges in the adjacency matrix.

    :return: The computed incidence matrix. It will have a shape of
        (n_nodes, n_edges).
    """
    # The adjacency matrix should be sparse, so get the indices of the edges.
    connected_node_indices = tf.where(triangular_adjacency)

    # Match each edge with one of the nodes connected by that edge. We refer
    # to the two nodes connected by each edge as "right" and "left",
    # for convenience.
    edge_indices = tf.range(connected_node_indices.shape[0], dtype=tf.int64)
    edges_with_left_nodes = tf.stack(
        [connected_node_indices[:, 0], edge_indices], axis=1
    )
    edges_with_right_nodes = tf.stack(
        [connected_node_indices[:, 1], edge_indices], axis=1
    )

    # We now have all the points that should go in the sparse binary
    # transformation matrix.
    edge_indicators = tf.ones_like(edge_indices, dtype=tf.float32)
    num_nodes = tf.to_int64(tf.shape(triangular_adjacency)[0])
    output_shape = tf.stack([num_nodes, num_edges])
    left_sparse = tf.SparseTensor(
        indices=edges_with_left_nodes, values=edge_indicators, dense_shape=output_shape
    )
    left_sparse = tf.sparse.reorder(left_sparse)
    right_sparse = tf.SparseTensor(
        indices=edges_with_right_nodes, values=edge_indicators, dense_shape=output_shape
    )
    right_sparse = tf.sparse.reorder(right_sparse)
    # Combine the matrices for the left and right nodes.
    combined_sparse = tf.sparse.maximum(left_sparse, right_sparse)

    return tf.sparse.to_dense(combined_sparse)
//  [python]
def _incidence_matrix_single(triangular_adjacency, *, num_edges):
    """
    Creates the corresponding incidence matrix for a graph with a particular
    adjacency matrix.

    :param triangular_adjacency: The binary adjacency matrix. Should have shape
        (n_nodes, n_nodes), and be triangular.
    :param num_edges: The number of edges to use in the output. Should be large
        enough to accommodate all the edges in the adjacency matrix.

    :return: The computed incidence matrix. It will have a shape of
        (n_nodes, n_edges).
    """
    # The adjacency matrix should be sparse, so get the indices of the edges.
    connected_node_indices = tf.where(triangular_adjacency)

    # Match each edge with one of the nodes connected by that edge. We refer
    # to the two nodes connected by each edge as "right" and "left",
    # for convenience.
    edge_indices = tf.range(connected_node_indices.shape[0], dtype=tf.int64)
    edges_with_left_nodes = tf.stack(
        [connected_node_indices[:, 0], edge_indices], axis=1
    )
    edges_with_right_nodes = tf.stack(
        [connected_node_indices[:, 1], edge_indices], axis=1
    )

    # We now have all the points that should go in the sparse binary
    # transformation matrix.
    edge_indicators = tf.ones_like(edge_indices, dtype=tf.float32)
    num_nodes = tf.cast(tf.shape(triangular_adjacency)[0], tf.int64)
    output_shape = tf.stack([num_nodes, num_edges])
    left_sparse = tf.SparseTensor(
        indices=edges_with_left_nodes, values=edge_indicators, dense_shape=output_shape
    )
    left_sparse = tf.sparse.reorder(left_sparse)
    right_sparse = tf.SparseTensor(
        indices=edges_with_right_nodes, values=edge_indicators, dense_shape=output_shape
    )
    right_sparse = tf.sparse.reorder(right_sparse)
    # Combine the matrices for the left and right nodes.
    combined_sparse = tf.sparse.maximum(left_sparse, right_sparse)

    return tf.sparse.to_dense(combined_sparse)
//  [python]
def resize_bilinear(images, size, output_dtype=tf.float32):
  """Returns resized images as output_type.

  Args:
    images: A tensor of size [batch, height_in, width_in, channels].
    size: A 1-D int32 Tensor of 2 elements: new_height, new_width. The new size
      for the images.
    output_dtype: The destination type.
  Returns:
    A tensor of size [batch, height_out, width_out, channels] as a dtype of
      output_dtype.
  """
  images = tf.image.resize_bilinear(images, size, align_corners=True)
  return tf.cast(images, dtype=output_dtype)
//  [python]
def scale_dimension(dim, scale):
  """Scales the input dimension.

  Args:
    dim: Input dimension (a scalar or a scalar Tensor).
    scale: The amount of scaling applied to the input.

  Returns:
    Scaled dimension.
  """
  if isinstance(dim, tf.Tensor):
    return tf.to_int32((tf.to_float(dim) - 1.0) * scale + 1.0)
  else:
    return int((float(dim) - 1.0) * scale + 1.0)
//  [python]
def scale_dimension(dim, scale):
  """Scales the input dimension.

  Args:
    dim: Input dimension (a scalar or a scalar Tensor).
    scale: The amount of scaling applied to the input.

  Returns:
    Scaled dimension.
  """
  if isinstance(dim, tf.Tensor):
    return tf.cast((tf.to_float(dim) - 1.0) * scale + 1.0, dtype=tf.int32)
  else:
    return int((float(dim) - 1.0) * scale + 1.0)
//  [python]
def train_step(self, data):
        x, y, sample_weight = self.unpack_training_data(data)

        with tf.GradientTape() as tape:
            (potentials, sequence_length, kernel), decoded_sequence, *_ = self(
                x, training=True, return_crf_internal=True
            )

            mask_value = 0
            # create a boolean mask with False for labels to be ignored
            special_mask = tf.not_equal(y, mask_value)
            special_mask = tf.to_float(special_mask)
            #tf.print(special_mask)
            #tf.print(potentials)
            # apply the mask to prediction vectors, it will put to 0
            # weights for label to ignore, normally neutralizing them for 
            # loss calculation
            potentials = tf.multiply(potentials, tf.expand_dims(special_mask, -1))
            #tf.print(potentials)

            # experiment: replace 0 by -100 because it's log-based potential
            #the_minus = tf.fill(tf.shape(potentials), -100.0)
            #potentials = tf.where(tf.equal(potentials, 0.0), the_minus, potentials)

            crf_loss = self.compute_crf_loss(
                potentials, sequence_length, kernel, y, sample_weight
            )
            loss = crf_loss + tf.reduce_sum(self.losses)
        gradients = tape.gradient(loss, self.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

        # Update metrics (includes the metric that tracks the loss)
        self.compiled_metrics.update_state(y, decoded_sequence)
        # Return a dict mapping metric names to current value
        orig_results = {m.name: m.result() for m in self.metrics}
        crf_results = {"loss": loss, "crf_loss": crf_loss}
        #crf_results = {"crf_loss": crf_loss}
        return {**orig_results, **crf_results}
//  [python]
def train_step(self, data):
        x, y, sample_weight = self.unpack_training_data(data)

        with tf.GradientTape() as tape:
            (potentials, sequence_length, kernel), decoded_sequence, *_ = self(
                x, training=True, return_crf_internal=True
            )

            mask_value = 0
            # create a boolean mask with False for labels to be ignored
            special_mask = tf.not_equal(y, mask_value)
            special_mask = tf.cast(special_mask, tf.float32)
            #tf.print(special_mask)
            #tf.print(potentials)
            # apply the mask to prediction vectors, it will put to 0
            # weights for label to ignore, normally neutralizing them for 
            # loss calculation
            potentials = tf.multiply(potentials, tf.expand_dims(special_mask, -1))
            #tf.print(potentials)

            # experiment: replace 0 by -100 because it's log-based potential
            #the_minus = tf.fill(tf.shape(potentials), -100.0)
            #potentials = tf.where(tf.equal(potentials, 0.0), the_minus, potentials)

            crf_loss = self.compute_crf_loss(
                potentials, sequence_length, kernel, y, sample_weight
            )
            loss = crf_loss + tf.reduce_sum(self.losses)
        gradients = tape.gradient(loss, self.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

        # Update metrics (includes the metric that tracks the loss)
        self.compiled_metrics.update_state(y, decoded_sequence)
        # Return a dict mapping metric names to current value
        orig_results = {m.name: m.result() for m in self.metrics}
        crf_results = {"loss": loss, "crf_loss": crf_loss}
        #crf_results = {"crf_loss": crf_loss}
        return {**orig_results, **crf_results}
//  [python]
def compose(transforms, interp_method='linear', shift_center=True, shape=None):
    """
    Compose a single transform from a series of transforms.

    Supports both dense and affine transforms, and returns a dense transform unless all
    inputs are affine. The list of transforms to compose should be in the order in which
    they would be individually applied to an image. For example, given transforms A, B,
    and C, to compose a single transform T, where T(x) = C(B(A(x))), the appropriate
    function call is:

    T = compose([A, B, C])

    Parameters:
        transforms: List or tuple of affine and/or dense transforms to compose.
        interp_method: Interpolation method. Must be 'linear' or 'nearest'.
        shift_center: Shift grid to image center when converting matrices to dense transforms.
        shape: ND output shape used for converting matrices to dense transforms. Includes only the
            N spatial dimensions. Only used once, if the rightmost transform is a matrix. If None
            or if the rightmost transform is a warp, the shape of the rightmost warp will be used.
            Incompatible with `shift_center=True`.

    Returns:
        Composed affine or dense transform.

    Notes:
        There used to be an argument for choosing between matrix ('ij') and Cartesian ('xy')
        indexing. Due to inconsistencies in how some functions and layers handled xy-indexing, we
        removed it in favor of default ij-indexing to minimize the potential for confusion.

    """
    if len(transforms) == 0:
        raise ValueError('Compose transform list cannot be empty')

    curr = None
    for next in reversed(transforms):

        if not tf.is_tensor(next) or not next.dtype.is_floating:
            next = t.cast(next, tf.float32)

        if curr is None:
            curr = next
            continue

        # Dense warp on left: interpolate. Shape will be ignored unless the current transform is a
        # matrix. Once the current transform is a warp field, it will stay a warp field.
        if not is_affine_shape(next.shape):
            if is_affine_shape(curr.shape):
                curr = affine_to_dense_shift(curr,
                                             shape=next.shape[:-1] if shape is None else shape,
                                             shift_center=shift_center)
            curr += transform(next, curr, interp_method=interp_method)

        # Matrix on left, dense warp on right: matrix-vector product.
        elif not is_affine_shape(curr.shape):
            curr = affine_to_dense_shift(next,
                                         shape=curr.shape[:-1],
                                         shift_center=shift_center,
                                         warp_right=curr)

        # No dense warp: matrix product.
        else:
            next = make_square_affine(next)
            curr = make_square_affine(curr)
            curr = tf.linalg.matmul(next, curr)[:-1]

    return curr
//  [python]
def compose(transforms, interp_method='linear', shift_center=True, shape=None):
    """
    Compose a single transform from a series of transforms.

    Supports both dense and affine transforms, and returns a dense transform unless all
    inputs are affine. The list of transforms to compose should be in the order in which
    they would be individually applied to an image. For example, given transforms A, B,
    and C, to compose a single transform T, where T(x) = C(B(A(x))), the appropriate
    function call is:

    T = compose([A, B, C])

    Parameters:
        transforms: List or tuple of affine and/or dense transforms to compose.
        interp_method: Interpolation method. Must be 'linear' or 'nearest'.
        shift_center: Shift grid to image center when converting matrices to dense transforms.
        shape: ND output shape used for converting matrices to dense transforms. Includes only the
            N spatial dimensions. Only used once, if the rightmost transform is a matrix. If None
            or if the rightmost transform is a warp, the shape of the rightmost warp will be used.
            Incompatible with `shift_center=True`.

    Returns:
        Composed affine or dense transform.

    Notes:
        There used to be an argument for choosing between matrix ('ij') and Cartesian ('xy')
        indexing. Due to inconsistencies in how some functions and layers handled xy-indexing, we
        removed it in favor of default ij-indexing to minimize the potential for confusion.

    """
    if len(transforms) == 0:
        raise ValueError('Compose transform list cannot be empty')

    curr = None
    for next in reversed(transforms):

        if not tf.is_tensor(next) or not next.dtype.is_floating:
            next = tf.cast(next, tf.float32)

        if curr is None:
            curr = next
            continue

        # Dense warp on left: interpolate. Shape will be ignored unless the current transform is a
        # matrix. Once the current transform is a warp field, it will stay a warp field.
        if not is_affine_shape(next.shape):
            if is_affine_shape(curr.shape):
                curr = affine_to_dense_shift(curr,
                                             shape=next.shape[:-1] if shape is None else shape,
                                             shift_center=shift_center)
            curr += transform(next, curr, interp_method=interp_method)

        # Matrix on left, dense warp on right: matrix-vector product.
        elif not is_affine_shape(curr.shape):
            curr = affine_to_dense_shift(next,
                                         shape=curr.shape[:-1],
                                         shift_center=shift_center,
                                         warp_right=curr)

        # No dense warp: matrix product.
        else:
            next = make_square_affine(next)
            curr = make_square_affine(curr)
            curr = tf.linalg.matmul(next, curr)[:-1]

    return curr
//  [python]
def __init__(self, initial_learning_rate, hidden_size, warmup_steps):
    """Initialize configuration of the learning rate schedule.

    Args:
      initial_learning_rate: A float, the initial learning rate.
      hidden_size: An integer, the model dimension in the hidden layers.
      warmup_steps: An integer, the number of steps required for linear warmup.
    """
    super(LearningRateSchedule, self).__init__()
    self.initial_learning_rate = initial_learning_rate
    self.hidden_size = hidden_size
    self.warmup_steps = t.cast(warmup_steps, t.float32)
//  [python]
def __init__(self, initial_learning_rate, hidden_size, warmup_steps):
    """Initialize configuration of the learning rate schedule.

    Args:
      initial_learning_rate: A float, the initial learning rate.
      hidden_size: An integer, the model dimension in the hidden layers.
      warmup_steps: An integer, the number of steps required for linear warmup.
    """
    super(LearningRateSchedule, self).__init__()
    self.initial_learning_rate = initial_learning_rate
    self.hidden_size = hidden_size
    self.warmup_steps = tf.cast(warmup_steps, tf.float32)
//  [python]
def affine_to_dense_shift(matrix, shape, shift_center=True, warp_right=None):
    """
    Convert N-dimensional (ND) matrix transforms to dense displacement fields.

    Algorithm:
        1. Build and (optionally) shift grid to center of image.
        2. Apply matrices to each index coordinate.
        3. Subtract grid.

    Parameters:
        matrix: Affine matrix of shape (..., M, N + 1), where M is N or N + 1. Can have any batch
            dimensions.
        shape: ND shape of the output space.
        shift_center: Shift grid to image center.
        warp_right: Right-compose the matrix transform with a displacement field of shape
            (..., *shape, N), with batch dimensions broadcastable to those of `matrix`.

    Returns:
        Dense shift (warp) of shape (..., *shape, N).

    Notes:
        There used to be an argument for choosing between matrix ('ij') and Cartesian ('xy')
        indexing. Due to inconsistencies in how some functions and layers handled xy-indexing, we
        removed it in favor of default ij-indexing to minimize the potential for confusion.

    """
    if isinstance(shape, (tf.compat.v1.Dimension, tf.TensorShape)):
        shape = shape.as_list()

    if not tf.is_tensor(matrix) or not matrix.dtype.is_floating:
        matrix = t.cast(matrix, tf.float32)

    # check input shapes
    ndims = len(shape)
    if matrix.shape[-1] != (ndims + 1):
        matdim = matrix.shape[-1] - 1
        raise ValueError(f'Affine ({matdim}D) does not match target shape ({ndims}D).')
    validate_affine_shape(matrix.shape)

    # coordinate grid
    mesh = (tf.range(s, dtype=matrix.dtype) for s in shape)
    if shift_center:
        mesh = (m - 0.5 * (s - 1) for m, s in zip(mesh, shape))
    mesh = [tf.reshape(m, shape=(-1,)) for m in tf.meshgrid(*mesh, indexing='ij')]
    mesh = tf.stack(mesh)  # N x nb_voxels
    out = mesh

    # optionally right-compose with warp field
    if warp_right is not None:
        if not tf.is_tensor(warp_right) or warp_right.dtype != matrix.dtype:
            warp_right = t.cast(warp_right, matrix.dtype)
        flat_shape = tf.concat((tf.shape(warp_right)[:-1 - ndims], (-1, ndims)), axis=0)
        warp_right = tf.reshape(warp_right, flat_shape)  # ... x nb_voxels x N
        out += tf.linalg.matrix_transpose(warp_right)  # ... x N x nb_voxels

    # compute locations, subtract grid to obtain shift
    out = matrix[..., :ndims, :-1] @ out + matrix[..., :ndims, -1:]  # ... x N x nb_voxels
    out = tf.linalg.matrix_transpose(out - mesh)  # ... x nb_voxels x N

    # restore shape
    shape = tf.concat((tf.shape(matrix)[:-2], (*shape, ndims)), axis=0)
    return tf.reshape(out, shape)