traning  a model `Falcon-7b instruct` and facing error
Author: rajveer43
Pull request: no

### System Info

Kaggle notebook, google colab

```
training_args = transformers.TrainingArguments(
      per_device_train_batch_size=4,
      gradient_accumulation_steps=4, #4
      num_train_epochs=6,
      learning_rate=2e-4,
      fp16=True,
      save_total_limit=3,
      logging_steps=500,
      output_dir="experiments",
      optim="paged_adamw_8bit",
      lr_scheduler_type="cosine",
      warmup_ratio=0.05,
      push_to_hub=True,
)

trainer = transformers.Trainer(
    model=model,
    train_dataset=train_data_transformed,
    args=training_args,
    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)
)
model.config.use_cache = False

trainer.train()
```


error:
```
RuntimeError: Inference tensors cannot be saved for backward. To work around you can make a clone to get a normal tensor and use it in autograd.
```

```
You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[27], line 1
----> 1 trainer.train()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1528, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1525 try:
   1526     # Disable progress bars when uploading models during checkpoints to avoid polluting stdout
   1527     hf_hub_utils.disable_progress_bars()
-> 1528     return inner_training_loop(
   1529         args=args,
   1530         resume_from_checkpoint=resume_from_checkpoint,
   1531         trial=trial,
   1532         ignore_keys_for_eval=ignore_keys_for_eval,
   1533     )
   1534 finally:
   1535     hf_hub_utils.enable_progress_bars()

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1854, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1851     self.control = self.callback_handler.on_step_begin(args, self.state, self.control)
   1853 with self.accelerator.accumulate(model):
-> 1854     tr_loss_step = self.training_step(model, inputs)
   1856 if (
   1857     args.logging_nan_inf_filter
   1858     and not is_torch_tpu_available()
   1859     and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))
   1860 ):
   1861     # if loss is nan or inf simply add the average of previous logged losses
   1862     tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)

File /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2732, in Trainer.training_step(self, model, inputs)
   2730         scaled_loss.backward()
   2731 else:
-> 2732     self.accelerator.backward(loss)
   2734 return loss.detach() / self.args.gradient_accumulation_steps

File /opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:1903, in Accelerator.backward(self, loss, **kwargs)
   1901     return
   1902 elif self.scaler is not None:
-> 1903     self.scaler.scale(loss).backward(**kwargs)
   1904 else:
   1905     loss.backward(**kwargs)

File /opt/conda/lib/python3.10/site-packages/torch/_tensor.py:487, in Tensor.backward(self, gradient, retain_graph, create_graph, inputs)
    477 if has_torch_function_unary(self):
    478     return handle_torch_function(
    479         Tensor.backward,
    480         (self,),
   (...)
    485         inputs=inputs,
    486     )
--> 487 torch.autograd.backward(
    488     self, gradient, retain_graph, create_graph, inputs=inputs
    489 )

File /opt/conda/lib/python3.10/site-packages/torch/autograd/__init__.py:200, in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    195     retain_graph = create_graph
    197 # The reason we repeat same the comment below is that
    198 # some Python versions print out the first line of a multi-line function
    199 # calls in the traceback and some print out the last line
--> 200 Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
    201     tensors, grad_tensors_, retain_graph, create_graph, inputs,
    202     allow_unreachable=True, accumulate_grad=True)

File /opt/conda/lib/python3.10/site-packages/torch/autograd/function.py:274, in BackwardCFunction.apply(self, *args)
    270     raise RuntimeError("Implementing both 'backward' and 'vjp' for a custom "
    271                        "Function is not allowed. You should only implement one "
    272                        "of them.")
    273 user_fn = vjp_fn if vjp_fn is not Function.vjp else backward_fn
--> 274 return user_fn(self, *args)

File /opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:141, in CheckpointFunction.backward(ctx, *args)
    137     detached_inputs = detach_variable(tuple(inputs))
    138     with torch.enable_grad(), \
    139          torch.cuda.amp.autocast(**ctx.gpu_autocast_kwargs), \
    140          torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):
--> 141         outputs = ctx.run_function(*detached_inputs)
    143 if isinstance(outputs, torch.Tensor):
    144     outputs = (outputs,)

File ~/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-7b-instruct/cf4b3c42ce2fdfe24f753f0f0d179202fea59c99/modeling_falcon.py:785, in FalconModel.forward.<locals>.create_custom_forward.<locals>.custom_forward(*inputs)
    783 def custom_forward(*inputs):
    784     # None for past_key_value
--> 785     return module(*inputs, use_cache=use_cache, output_attentions=output_attentions)

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    163         output = module._old_forward(*args, **kwargs)
    164 else:
--> 165     output = module._old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File ~/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-7b-instruct/cf4b3c42ce2fdfe24f753f0f0d179202fea59c99/modeling_falcon.py:453, in FalconDecoderLayer.forward(self, hidden_states, alibi, attention_mask, layer_past, head_mask, use_cache, output_attentions)
    450     attention_layernorm_out = self.input_layernorm(hidden_states)
    452 # Self attention.
--> 453 attn_outputs = self.self_attention(
    454     attention_layernorm_out,
    455     layer_past=layer_past,
    456     attention_mask=attention_mask,
    457     alibi=alibi,
    458     head_mask=head_mask,
    459     use_cache=use_cache,
    460     output_attentions=output_attentions,
    461 )
    463 attention_output = attn_outputs[0]
    465 if not self.config.new_decoder_architecture:

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File /opt/conda/lib/python3.10/site-packages/accelerate/hooks.py:165, in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    163         output = module._old_forward(*args, **kwargs)
    164 else:
--> 165     output = module._old_forward(*args, **kwargs)
    166 return module._hf_hook.post_forward(module, output)

File ~/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-7b-instruct/cf4b3c42ce2fdfe24f753f0f0d179202fea59c99/modeling_falcon.py:307, in FalconAttention.forward(self, hidden_states, alibi, attention_mask, layer_past, head_mask, use_cache, output_attentions)
    304 value_layer = value_layer.transpose(1, 2).reshape(batch_size * num_kv_heads, query_length, self.head_dim)
    306 past_kv_length = 0 if layer_past is None else layer_past[0].shape[1]
--> 307 query_layer, key_layer = self.maybe_rotary(query_layer, key_layer, past_kv_length)
    309 if layer_past is not None:
    310     past_key, past_value = layer_past

File /opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.cache/huggingface/modules/transformers_modules/tiiuae/falcon-7b-instruct/cf4b3c42ce2fdfe24f753f0f0d179202fea59c99/modeling_falcon.py:108, in FalconRotaryEmbedding.forward(self, query, key, past_key_values_length)
    106 batch, seq_len, head_dim = query.shape
    107 cos, sin = self.cos_sin(seq_len, past_key_values_length, query.device, query.dtype)
--> 108 return (query * cos) + (rotate_half(query) * sin), (key * cos) + (rotate_half(key) * sin)

RuntimeError: Inference tensors cannot be saved for backward. To work around you can make a clone to get a normal tensor and use it in autograd.
```

how can I solve this?






### Who can help?

_No response_

### Information

- [ ] The official example scripts
- [ ] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

-

### Expected behavior

-

@ArthurZucker 

Comments:
The snippet is not full 

> The snippet is not full

I have updated the snippet 

I can't guess the model you are using, is it quantized did you use from pretrained, which checkpoint, trust remote code? ? etc

```
MODEL_NAME = "tiiuae/falcon-7b-instruct"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    device_map="auto",
    trust_remote_code=True,
    quantization_config=bnb_config
)

tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer.pad_token = tokenizer.eos_token

model.gradient_checkpointing_enable()
model = prepare_model_for_kbit_training(model)

config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["query_key_value"],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    
)
# model.add_adapter(config)
model = get_peft_model(model, config)
print_trainable_parameters(model)
```




I am using it from pretranied for QNA task , for medical data




I think I found the issue! check and see if your transformers version is up to date because this problem occurs when the Trainer displaces the model to the CPU to save it, when the saving ends trainer can't train with TPU anymore because the whole model is in the CPU now! (but this problem is fixed) Check you're transformers version and use this to update it:
!pip install -U git+https://github.com/huggingface/transformers.git

Thanks a bunch! @Aliync, It worked! 

I was already on the latest version of Transformers and even after 
`!pip install -U git+https://github.com/huggingface/transformers.git`
it didn't work, am I missing something?