/usr/local/lib/python3.10/dist-packages/accelerate/accelerator.py:494: FutureWarning: `torch.cuda.amp.GradScaler(args...)` is deprecated. Please use `torch.amp.GradScaler('cuda', args...)` instead.
  self.scaler = torch.cuda.amp.GradScaler(**kwargs)
/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
 [1250/1250 06:03, Epoch 1/1]
Epoch	Training Loss	Validation Loss
 [250/250 04:16]
/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
Preds shape: (250, 128)
Preds sample: [    0     3    15  1169  3010    11   306    18 13261  1045  7686    87
  6644  2052     5   264   572   744    31    17  5987   131   453  5792
   323    11   572    62   217   126  1791   369    11   625  1791   281
    58 10403   841    19  3823    30     3     9  4833    13   335   770
  3740    41 23180   182 30238    21   204   727  3381   502   137   328
  4032    48   139   796  2055 15841   224    38  3599  2055     6     3
    15  1169  3010    11   306    18 13261  1045  7686    87  6644  2052
     5   852   497    79    33 11252  4367    11  2396   167    13    70
  2055    36  1213    16  3599  2055     6     3    15  1169  3010    11
   306    18 13261  1045  7686    87  6644  2052     5   852   497    79
    33 11252  4367    11  2396   167    13    70]
Max token ID in preds: 31762
Min token ID in preds: -100
Tokenizer vocab size: 32000
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-26-1513d96d834a> in <cell line: 31>()
     29 
     30 # Start training the model
---> 31 trainer.train()

15 frames
/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1936                 hf_hub_utils.enable_progress_bars()
   1937         else:
-> 1938             return inner_training_loop(
   1939                 args=args,
   1940                 resume_from_checkpoint=resume_from_checkpoint,

/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in _inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   2374 
   2375             self.control = self.callback_handler.on_epoch_end(args, self.state, self.control)
-> 2376             self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval)
   2377 
   2378             if DebugOption.TPU_METRICS_DEBUG in self.args.debug:

/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in _maybe_log_save_evaluate(self, tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval)
   2802         metrics = None
   2803         if self.control.should_evaluate:
-> 2804             metrics = self._evaluate(trial, ignore_keys_for_eval)
   2805 
   2806         if self.control.should_save:

/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in _evaluate(self, trial, ignore_keys_for_eval, skip_scheduler)
   2759 
   2760     def _evaluate(self, trial, ignore_keys_for_eval, skip_scheduler=False):
-> 2761         metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)
   2762         self._report_to_hp_search(trial, self.state.global_step, metrics)
   2763 

/usr/local/lib/python3.10/dist-packages/transformers/trainer_seq2seq.py in evaluate(self, eval_dataset, ignore_keys, metric_key_prefix, **gen_kwargs)
    178         self.gather_function = self.accelerator.gather
    179         self._gen_kwargs = gen_kwargs
--> 180         return super().evaluate(eval_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix)
    181 
    182     def predict(

/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in evaluate(self, eval_dataset, ignore_keys, metric_key_prefix)
   3664 
   3665         eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
-> 3666         output = eval_loop(
   3667             eval_dataloader,
   3668             description="Evaluation",

/usr/local/lib/python3.10/dist-packages/transformers/trainer.py in evaluation_loop(self, dataloader, description, prediction_loss_only, ignore_keys, metric_key_prefix)
   3954                 )
   3955             else:
-> 3956                 metrics = self.compute_metrics(EvalPrediction(predictions=all_preds, label_ids=all_labels))
   3957         elif metrics is None:
   3958             metrics = {}

<ipython-input-25-38909425dac7> in compute_metrics(eval_pred)
     54 
     55     # Decode the predicted tokens into text
---> 56     decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
     57 
     58     # Replace -100 in the labels as we can't decode them

/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py in batch_decode(self, sequences, skip_special_tokens, clean_up_tokenization_spaces, **kwargs)
   3974             `List[str]`: The list of decoded sentences.
   3975         """
-> 3976         return [
   3977             self.decode(
   3978                 seq,

/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py in <listcomp>(.0)
   3975         """
   3976         return [
-> 3977             self.decode(
   3978                 seq,
   3979                 skip_special_tokens=skip_special_tokens,

/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py in decode(self, token_ids, skip_special_tokens, clean_up_tokenization_spaces, **kwargs)
   4014         token_ids = to_py_obj(token_ids)
   4015 
-> 4016         return self._decode(
   4017             token_ids=token_ids,
   4018             skip_special_tokens=skip_special_tokens,

/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils.py in _decode(self, token_ids, skip_special_tokens, clean_up_tokenization_spaces, spaces_between_special_tokens, **kwargs)
   1079         self._decode_use_source_tokenizer = kwargs.pop("use_source_tokenizer", False)
   1080 
-> 1081         filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens)
   1082         legacy_added_tokens = set(self._added_tokens_encoder.keys()) - set(self.all_special_tokens) | {
   1083             token for token in self.additional_special_tokens if self.convert_tokens_to_ids(token) >= self.vocab_size

/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils.py in convert_ids_to_tokens(self, ids, skip_special_tokens)
   1060                 tokens.append(self._added_tokens_decoder[index].content)
   1061             else:
-> 1062                 tokens.append(self._convert_id_to_token(index))
   1063         return tokens
   1064 

/usr/local/lib/python3.10/dist-packages/transformers/models/t5/tokenization_t5.py in _convert_id_to_token(self, index)
    404     def _convert_id_to_token(self, index):
    405         """Converts an index (integer) in a token (str) using the vocab."""
--> 406         token = self.sp_model.IdToPiece(index)
    407         return token
    408 

/usr/local/lib/python3.10/dist-packages/sentencepiece/__init__.py in _batched_func(self, arg)
   1043       return [_func(self, n) for n in arg]
   1044     else:
-> 1045       return _func(self, arg)
   1046 
   1047   setattr(classname, name, _batched_func)

/usr/local/lib/python3.10/dist-packages/sentencepiece/__init__.py in _func(v, n)
   1036   def _func(v, n):
   1037     if type(n) is int and (n < 0 or n >= v.piece_size()):
-> 1038       raise IndexError('piece id is out of range.')
   1039     return func(v, n)
   1040 

IndexError: piece id is out of range.