Error in fine tuning Roberta for QA
Author: houdaM97
Pull request: no
Labels: name: wontfix

## ❓ Questions & Help

<!-- A clear and concise description of the question. -->
Hi, 
I tried to fine tune RobertaModel for question answering task, i implemented TFRobertaForQuestionAnswering but when i run the training script i got this error:

tensorflow.python.framework.errors_impl.InvalidArgumentError: indices[0,11] = 1 is not in [0, 1) [Op:ResourceGather] name: tf_roberta_for_question_answering/tf_roberta_model/roberta/embeddings/token_type_embeddings/embedding_lookup/

Here is my class TFRobertaForQuestionAnswering
```python
from transformers import TFRobertaPreTrainedModel, RobertaConfig, TFRobertaModel
import tensorflow as tf


class TFRobertaForQuestionAnswering(TFRobertaPreTrainedModel):

    config_class = RobertaConfig
    #pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP
    base_model_prefix = "roberta"

    def __init__(self, config, *inputs, **kwargs):
        config.vocab_size = config.vocab_size + 2
        super().__init__(config, *inputs, **kwargs)

        def get_initializer(initializer_range=0.02):
            """Creates a `tf.initializers.truncated_normal` with the given range.
            Args:
                initializer_range: float, initializer range for stddev.
            Returns:
                TruncatedNormal initializer with stddev = `initializer_range`.
            """
            return tf.keras.initializers.TruncatedNormal(stddev=initializer_range)

        self.num_labels = config.num_labels
        self.roberta = TFRobertaModel(config)
        self.qa_outputs = tf.keras.layers.Dense(
            config.num_labels, kernel_initializer=get_initializer(config.initializer_range), name="qa_outputs")

    def call(
        self,
        input_ids,
        start_positions=None,
        end_positions=None,
        ** kwargs
    ):

        outputs = self.roberta(
            input_ids,
            ** kwargs
        )

        sequence_output = outputs[0]

        logits = self.qa_outputs(sequence_output)
        start_logits, end_logits = tf.split(logits, 2, axis=-1)
        start_logits = tf.squeeze(start_logits, -1)
        end_logits = tf.squeeze(end_logits, -1)

        outputs = (start_logits, end_logits,) + outputs[2:]
        if start_positions is not None and end_positions is not None:
            # If we are on multi-GPU, split add a dimension
            if len(tf.size(start_positions)) > 1:
                start_positions = tf.squeeze(start_positions, -1)
            if len(tf.size(end_positions)) > 1:
                end_positions = tf.squeeze(end_positions, -1)
            # sometimes the start/end positions are outside our model inputs, we ignore these terms
            #ignored_index = tf.size(start_logits, 1)
            #with tf.Session() as sess:
            #    scalar = ignored_index.eval()
            #tf.clip_by_value(start_positions, 0, ignored_index)
            #tf.clip_by_value(end_positions, 0, ignored_index)

            loss_fct = tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True)
            start_loss = loss_fct(start_logits, start_positions)
            end_loss = loss_fct(end_logits, end_positions)
            total_loss = (start_loss + end_loss) / 2
            outputs = (total_loss,) + outputs

        return outputs  # (loss), start_logits, end_logits, (hidden_states), (attentions)
```


and here is my training script:

```python
import tensorflow as tf
from transformers import squad_convert_examples_to_features, SquadV2Processor, RobertaTokenizer
from modelRoberta import TFRobertaForQuestionAnswering
from pathlib import Path
import six
import numpy as np

MAX_QUERY_LENGTH = 64
MAX_SEQ_LENGTH = 384
MAX_DOC_STRIDE = 128
MAX_ANSWER_LENGTH = 64
N_TOK_FOR_CONTEXT = 20

def get_shape_list(tensor):
  shape = tensor.shape.as_list()

  non_static_indexes = []
  for (index, dim) in enumerate(shape):
    if dim is None:
      non_static_indexes.append(index)

  if not non_static_indexes:
    return shape

  dyn_shape = tf.shape(tensor)
  for index in non_static_indexes:
    shape[index] = dyn_shape[index]
  return shape

tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
model = TFRobertaForQuestionAnswering.from_pretrained("roberta-base")

squad = SquadV2Processor()

train_examples = squad.get_train_examples(data_dir=Path(__file__).parent, filename="train.json")
test_examples = squad.get_dev_examples(data_dir=Path(__file__).parent, filename="test.json")


train_dataset = squad_convert_examples_to_features(train_examples[:1], tokenizer=tokenizer, max_seq_length=MAX_SEQ_LENGTH,
                                                   doc_stride=MAX_DOC_STRIDE, max_query_length=MAX_QUERY_LENGTH,
                                                   is_training=True, return_dataset='tf')
test_dataset = squad_convert_examples_to_features(test_examples[:1], tokenizer=tokenizer, max_seq_length=MAX_SEQ_LENGTH,
                                                   doc_stride=MAX_DOC_STRIDE, max_query_length=MAX_QUERY_LENGTH,
                                                   is_training=False, return_dataset='tf')

optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0)
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')

batch_size = 1
train_dataset = train_dataset.shuffle(buffer_size=1024).batch(batch_size)
epochs = 3
for epoch in range(epochs):
  print('Start of epoch %d' % (epoch,))

  for step, batch in enumerate(train_dataset):

    with tf.GradientTape() as tape:

          inputs_batch = {
              "inputs_ids": batch[0]['input_ids'],
              "token_type_ids": batch[0]['token_type_ids'],
          }

          start_position = batch[1]['start_position']
          end_position = batch[1]['end_position']
          seq_length = get_shape_list(inputs_batch['inputs_ids'])[1]
          start_positions = tf.one_hot(start_position, on_value=1.0, off_value=0.0, depth=seq_length, dtype=tf.float32)
          end_positions = tf.one_hot(end_position,on_value=1.0, off_value=0.0, depth=seq_length, dtype=tf.float32)

          outputs = model(inputs_batch['inputs_ids'], token_type_ids= inputs_batch['token_type_ids'], training=True)  # Logits for this minibatch

          start_logits, end_logits = outputs[:2]
          start_logits = tf.nn.log_softmax(start_logits, axis=-1)
          end_logits = tf.nn.log_softmax(end_logits, axis=-1)

          seq_height = get_shape_list(inputs_batch['inputs_ids'])[0]
          start_logits = tf.keras.backend.reshape(start_logits, shape=(seq_height*seq_length, 1))
          end_logits = tf.keras.backend.reshape(end_logits, shape=(seq_height * seq_length, 1))
          start_positions = tf.keras.backend.reshape(start_positions, shape=(seq_height * seq_length, 1))
          end_positions = tf.keras.backend.reshape(end_positions, shape=(seq_height * seq_length, 1))
          start_loss = -tf.reduce_mean(tf.reduce_sum(start_positions * start_logits, axis=-1))
          end_loss = -tf.reduce_mean(tf.reduce_sum(end_positions * end_logits, axis=-1))
          loss_value = (start_loss + end_loss) / 2


         grads = tape.gradient(loss_value, model.trainable_weights)

        optimizer.apply_gradients(zip(grads, model.trainable_weights))

    if step % 200 == 0:
        print('Training loss (for one batch) at step %s: %s' % (step, float(loss_value)))
        print('Seen so far: %s samples' % ((step + 1) * batch_size))

context = "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse ('Norman' comes from 'Norseman') raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries."
question = "In what country is Normandy located?"

en_plus = tokenizer.encode_plus(context, question, add_special_tokens=True)
en = en_plus['input_ids']
token_type_ids = en_plus['token_type_ids']
input_ids = tf.constant([en])

segments_tensors = tf.constant([token_type_ids])

outputs = model(input_ids)

start_scores, end_scores = outputs[:2]

ss = tf.argmax(start_scores.numpy()[0]).numpy()
es = tf.argmax(end_scores.numpy()[0]).numpy()

answer = tokenizer.decode(en[ss: es+1], clean_up_tokenization_spaces=True)
print(ss)
print(es)
print(answer)

model.save_pretrained('./save/')
```

Thanks in advance for helping me.

Comments:
Please [format your code correctly](https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks). Now it is very unreadable.

Thank you for mentioning this, it's done!

Something seems to have gone wrong. Can you check? There is a line "and here is my training script:" that is in the code block but shouldn't be. Also the last line.

yeah i corrected it

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.