Cuda out of memory on 2x 32 Gb V100

Sourabh_Yadav: <p>pretty much question but I am loading model in 8 bit and using peft. also batch size =1. and everything is set to low a per my understanding.</p>
<p><strong>::Also this script and settings worked for a different dataset yesterday .</strong></p>
<p>device = “cuda” if torch.cuda.is_available() else “cpu”</p>
<p>model_id = “Qwen/Qwen2.5-VL-3B-Instruct”</p>
<p>EPOCHS = 1</p>
<p>BATCH_SIZE = 1</p>
<p>GRADIENT_CHECKPOINTING = True</p>
<p>USE_REENTRANT = False</p>
<p>LEARNING_RATE = 1e-5</p>
<p>LOGGING_STEPS = 50</p>
<p>EVAL_STEPS = 50</p>
<p>SAVE_STEPS = 50</p>
<p>EVAL_STRATEGY = “steps”</p>
<p>SAVE_STRATEGY = “steps”</p>
<p>METRIC_FOR_BEST_MODEL = “eval_loss”</p>
<p>LOAD_BEST_MODEL_AT_END = True</p>
<p>MAX_GRAD_NORM = 1.0</p>
<p>DATASET_KWARGS = {“skip_prepare_dataset”: True}</p>
<p>REMOVE_UNUSED_COLUMNS = False</p>
<p>MAX_SEQ_LEN = 128</p>
<p>NUM_STEPS = (58 // BATCH_SIZE) * EPOCHS ### Print(len(train_dataset) to get this ###</p>
<p>print(f"NUM_STEPS: {NUM_STEPS}")</p>
<p>from transformers import Trainer, TrainingArguments<br>
from transformers.optimization import AdamW<br>
import torch</p>
<p>device = “cuda” if torch.cuda.is_available() else “cpu”</p>
<p>optimizer = AdamW(model.parameters(), lr=LEARNING_RATE)</p>
<p>training_args = TrainingArguments(<br>
output_dir=“./results”,<br>
learning_rate=LEARNING_RATE,<br>
per_device_train_batch_size=BATCH_SIZE,<br>
num_train_epochs=EPOCHS,<br>
weight_decay=0.01,<br>
logging_dir=“./logs”,<br>
logging_steps=LOGGING_STEPS,<br>
save_steps=SAVE_STEPS,<br>
gradient_checkpointing=GRADIENT_CHECKPOINTING,<br>
max_grad_norm=MAX_GRAD_NORM,<br>
remove_unused_columns=REMOVE_UNUSED_COLUMNS,<br>
save_strategy=SAVE_STRATEGY,<br>
no_cuda=False<br>
)</p>
<p>def batch_collate_fn(examples):<br>
batched_inputs = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
for example in examples:<br>
texts = [processor.apply_chat_template(example, tokenize=False)]<br>
image_inputs = [example[1][“content”][0][“image”]]<br>
inputs = processor(<br>
text=texts,<br>
images=image_inputs,<br>
return_tensors=“pt”,<br>
padding=True<br>
)<br>
inputs[“labels”] = inputs[“input_ids”].clone()<br>
batched_inputs.append(inputs)</p>
<pre><code># Combine all batches
combined_inputs = {
    key: torch.cat([b[key] for b in batched_inputs]) 
    for key in batched_inputs[0].keys()
}
return combined_inputs
</code></pre>
<p>trainer = Trainer(<br>
model=model,<br>
args=training_args,<br>
train_dataset=train_dataset,<br>
# eval_dataset=eval_dataset,<br>
optimizers=(optimizer, None),<br>
data_collator=batch_collate_fn<br>
)</p>
<h1><a name="p-464945-printstarting-evaluation-1" class="anchor" href="#p-464945-printstarting-evaluation-1"></a>print(“Starting evaluation…”)</h1>
<h1><a name="p-464945-metric-trainerevaluate-2" class="anchor" href="#p-464945-metric-trainerevaluate-2"></a>metric = trainer.evaluate()</h1>
<h1><a name="p-464945-printfevaluation-metrics-metric-3" class="anchor" href="#p-464945-printfevaluation-metrics-metric-3"></a>print(f"Evaluation metrics: {metric}")</h1>
<p>print(“-”*30)</p>
<p>print(“Starting training…”)<br>
trainer.train()</p>
<hr>
<p>OutOfMemoryError                          Traceback (most recent call last)<br>
Cell In[8], line 62<br>
59 print(“-”*30)<br>
61 print(“Starting training…”)<br>
—&gt; 62 trainer.train()</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\trainer.py:2184, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)<br>
2182         hf_hub_utils.enable_progress_bars()<br>
2183 else:<br>
 → 2184     return inner_training_loop(<br>
2185         args=args,<br>
2186         resume_from_checkpoint=resume_from_checkpoint,<br>
2187         trial=trial,<br>
2188         ignore_keys_for_eval=ignore_keys_for_eval,<br>
2189     )</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\trainer.py:2490, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)<br>
2483 context = (<br>
2484     functools.partial(self.accelerator.no_sync, model=model)<br>
2485     if i != len(batch_samples) - 1<br>
2486     and self.accelerator.distributed_type != DistributedType.DEEPSPEED<br>
2487     else contextlib.nullcontext<br>
2488 )<br>
2489 with context():<br>
 → 2490     tr_loss_step = self.training_step(model, inputs, num_items_in_batch)<br>
2492 if (<br>
2493     args.logging_nan_inf_filter<br>
2494     and not is_torch_xla_available()<br>
2495     and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))<br>
2496 ):<br>
2497     # if loss is nan or inf simply add the average of previous logged losses<br>
2498     tr_loss = tr_loss + tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\trainer.py:3598, in Trainer.training_step(self, model, inputs, num_items_in_batch)<br>
3595     return loss_mb.reduce_mean().detach().to(self.args.device)<br>
3597 with self.compute_loss_context_manager():<br>
 → 3598     loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)<br>
3600 del inputs<br>
3601 if (<br>
3602     self.args.torch_empty_cache_steps is not None<br>
3603     and self.state.global_step % self.args.torch_empty_cache_steps == 0<br>
3604 ):</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\trainer.py:3659, in Trainer.compute_loss(self, model, inputs, return_outputs, num_items_in_batch)<br>
3657         loss_kwargs[“num_items_in_batch”] = num_items_in_batch<br>
3658     inputs = {**inputs, **loss_kwargs}<br>
 → 3659 outputs = model(**inputs)<br>
3660 # Save past state if it exists<br>
3661 # TODO: this needs to be fixed and made cleaner later.<br>
3662 if self.args.past_index &gt;= 0:</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1739, in Module._wrapped_call_impl(self, *args, **kwargs)<br>
1737     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]<br>
1738 else:<br>
 → 1739     return self._call_impl(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1750, in Module._call_impl(self, *args, **kwargs)<br>
1745 # If we don’t have any hooks, we want to skip the rest of the logic in<br>
1746 # this function, and just call forward.<br>
1747 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks<br>
1748         or _global_backward_pre_hooks or _global_backward_hooks<br>
1749         or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1750     return forward_call(*args, **kwargs)<br>
1752 result = None<br>
1753 called_always_called_hooks = set()</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\peft\peft_model.py:1719, in PeftModelForCausalLM.forward(self, input_ids, attention_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict, task_ids, **kwargs)<br>
1717     with self._enable_peft_forward_hooks(**kwargs):<br>
1718         kwargs = {k: v for k, v in kwargs.items() if k not in self.special_peft_forward_args}<br>
 → 1719         return self.base_model(<br>
1720             input_ids=input_ids,<br>
1721             attention_mask=attention_mask,<br>
1722             inputs_embeds=inputs_embeds,<br>
1723             labels=labels,<br>
1724             output_attentions=output_attentions,<br>
1725             output_hidden_states=output_hidden_states,<br>
1726             return_dict=return_dict,<br>
1727             **kwargs,<br>
1728         )<br>
1730 batch_size = _get_batch_size(input_ids, inputs_embeds)<br>
1731 if attention_mask is not None:<br>
1732     # concat prompt attention mask</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1739, in Module._wrapped_call_impl(self, *args, **kwargs)<br>
1737     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]<br>
1738 else:<br>
 → 1739     return self._call_impl(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1750, in Module._call_impl(self, *args, **kwargs)<br>
1745 # If we don’t have any hooks, we want to skip the rest of the logic in<br>
1746 # this function, and just call forward.<br>
1747 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks<br>
1748         or _global_backward_pre_hooks or _global_backward_hooks<br>
1749         or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1750     return forward_call(*args, **kwargs)<br>
1752 result = None<br>
1753 called_always_called_hooks = set()</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\peft\tuners\tuners_utils.py:197, in BaseTuner.forward(self, *args, **kwargs)<br>
196 def forward(self, *args: Any, **kwargs: Any):<br>
 → 197     return self.model.forward(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\accelerate\hooks.py:170, in add_hook_to_module..new_forward(module, *args, **kwargs)<br>
168         output = module._old_forward(*args, **kwargs)<br>
169 else:<br>
 → 170     output = module._old_forward(*args, **kwargs)<br>
171 return module._hf_hook.post_forward(module, output)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\models\qwen2_5_vl\modeling_qwen2_5_vl.py:1740, in Qwen2_5_VLForConditionalGeneration.forward(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, pixel_values, pixel_values_videos, image_grid_thw, video_grid_thw, rope_deltas, cache_position, second_per_grid_ts)<br>
1738 if pixel_values is not None:<br>
1739     pixel_values = pixel_values.type(self.visual.dtype)<br>
 → 1740     image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)<br>
1741     n_image_tokens = (input_ids == self.config.image_token_id).sum().item()<br>
1742     n_image_features = image_embeds.shape[0]</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1739, in Module._wrapped_call_impl(self, *args, **kwargs)<br>
1737     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]<br>
1738 else:<br>
 → 1739     return self._call_impl(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1750, in Module._call_impl(self, *args, **kwargs)<br>
1745 # If we don’t have any hooks, we want to skip the rest of the logic in<br>
1746 # this function, and just call forward.<br>
1747 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks<br>
1748         or _global_backward_pre_hooks or _global_backward_hooks<br>
1749         or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1750     return forward_call(*args, **kwargs)<br>
1752 result = None<br>
1753 called_always_called_hooks = set()</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\accelerate\hooks.py:170, in add_hook_to_module..new_forward(module, *args, **kwargs)<br>
168         output = module._old_forward(*args, **kwargs)<br>
169 else:<br>
 → 170     output = module._old_forward(*args, **kwargs)<br>
171 return module._hf_hook.post_forward(module, output)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\models\qwen2_5_vl\modeling_qwen2_5_vl.py:497, in Qwen2_5_VisionTransformerPretrainedModel.forward(self, hidden_states, grid_thw)<br>
495     cu_seqlens_now = cu_window_seqlens<br>
496 if self.gradient_checkpointing and self.training:<br>
 → 497     hidden_states = self._gradient_checkpointing_func(<br>
498         blk.<strong>call</strong>, hidden_states, cu_seqlens_now, rotary_pos_emb<br>
499     )<br>
500 else:<br>
501     hidden_states = blk(<br>
502         hidden_states,<br>
503         cu_seqlens=cu_seqlens_now,<br>
504         rotary_pos_emb=rotary_pos_emb,<br>
505     )</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch_compile.py:32, in _disable_dynamo..inner(*args, **kwargs)<br>
29     disable_fn = torch._dynamo.disable(fn, recursive)<br>
30     fn.__dynamo_disable = disable_fn<br>
—&gt; 32 return disable_fn(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch_dynamo\eval_frame.py:745, in DisableContext.<strong>call</strong>.._fn(*args, **kwargs)<br>
741 prior_skip_guard_eval_unsafe = set_skip_guard_eval_unsafe(<br>
742     _is_skip_guard_eval_unsafe_stance()<br>
743 )<br>
744 try:<br>
 → 745     return fn(*args, **kwargs)<br>
746 finally:<br>
747     _maybe_set_eval_frame(prior)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\utils\checkpoint.py:489, in checkpoint(function, use_reentrant, context_fn, determinism_check, debug, *args, **kwargs)<br>
484     if context_fn is not noop_context_fn or debug is not False:<br>
485         raise ValueError(<br>
486             "Passing <code>context_fn</code> or <code>debug</code> is only supported when "<br>
487             “use_reentrant=False.”<br>
488         )<br>
 → 489     return CheckpointFunction.apply(function, preserve, *args)<br>
490 else:<br>
491     gen = _checkpoint_without_reentrant_generator(<br>
492         function, preserve, context_fn, determinism_check, debug, *args, **kwargs<br>
493     )</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\autograd\function.py:575, in Function.apply(cls, *args, **kwargs)<br>
572 if not torch._C._are_functorch_transforms_active():<br>
573     # See NOTE: [functorch vjp and autograd interaction]<br>
574     args = _functorch.utils.unwrap_dead_wrappers(args)<br>
 → 575     return super().apply(*args, **kwargs)  # type: ignore[misc]<br>
577 if not is_setup_ctx_defined:<br>
578     raise RuntimeError(<br>
579         "In order to use an autograd.Function with functorch transforms "<br>
580         "(vmap, grad, jvp, jacrev, …), it must override the setup_context "<br>
581         "staticmethod. For more details, please see "<br>
582         “<a href="https://pytorch.org/docs/main/notes/extending.func.html" class="inline-onebox" rel="noopener nofollow ugc">Extending torch.func with autograd.Function — PyTorch main documentation</a>”<br>
583     )</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\utils\checkpoint.py:264, in CheckpointFunction.forward(ctx, run_function, preserve_rng_state, *args)<br>
261 ctx.save_for_backward(*tensor_inputs)<br>
263 with torch.no_grad():<br>
 → 264     outputs = run_function(*args)<br>
265 return outputs</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1739, in Module._wrapped_call_impl(self, *args, **kwargs)<br>
1737     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]<br>
1738 else:<br>
 → 1739     return self._call_impl(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1750, in Module._call_impl(self, *args, **kwargs)<br>
1745 # If we don’t have any hooks, we want to skip the rest of the logic in<br>
1746 # this function, and just call forward.<br>
1747 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks<br>
1748         or _global_backward_pre_hooks or _global_backward_hooks<br>
1749         or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1750     return forward_call(*args, **kwargs)<br>
1752 result = None<br>
1753 called_always_called_hooks = set()</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\accelerate\hooks.py:170, in add_hook_to_module..new_forward(module, *args, **kwargs)<br>
168         output = module._old_forward(*args, **kwargs)<br>
169 else:<br>
 → 170     output = module._old_forward(*args, **kwargs)<br>
171 return module._hf_hook.post_forward(module, output)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\models\qwen2_5_vl\modeling_qwen2_5_vl.py:297, in Qwen2_5_VLVisionBlock.forward(self, hidden_states, cu_seqlens, rotary_pos_emb)<br>
296 def forward(self, hidden_states, cu_seqlens, rotary_pos_emb) → torch.Tensor:<br>
 → 297     hidden_states = hidden_states + self.attn(<br>
298         self.norm1(hidden_states),<br>
299         cu_seqlens=cu_seqlens,<br>
300         rotary_pos_emb=rotary_pos_emb,<br>
301     )<br>
302     hidden_states = hidden_states + self.mlp(self.norm2(hidden_states))<br>
303     return hidden_states</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1739, in Module._wrapped_call_impl(self, *args, **kwargs)<br>
1737     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]<br>
1738 else:<br>
 → 1739     return self._call_impl(*args, **kwargs)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\torch\nn\modules\module.py:1750, in Module._call_impl(self, *args, **kwargs)<br>
1745 # If we don’t have any hooks, we want to skip the rest of the logic in<br>
1746 # this function, and just call forward.<br>
1747 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks<br>
1748         or _global_backward_pre_hooks or _global_backward_hooks<br>
1749         or _global_forward_hooks or _global_forward_pre_hooks):<br>
 → 1750     return forward_call(*args, **kwargs)<br>
1752 result = None<br>
1753 called_always_called_hooks = set()</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\accelerate\hooks.py:170, in add_hook_to_module..new_forward(module, *args, **kwargs)<br>
168         output = module._old_forward(*args, **kwargs)<br>
169 else:<br>
 → 170     output = module._old_forward(*args, **kwargs)<br>
171 return module._hf_hook.post_forward(module, output)</p>
<p>File c:\ProgramData\anaconda3\envs\qwen\lib\site-packages\transformers\models\qwen2_5_vl\modeling_qwen2_5_vl.py:272, in Qwen2_5_VLVisionSdpaAttention.forward(self, hidden_states, cu_seqlens, rotary_pos_emb)<br>
270 k = k.transpose(0, 1)<br>
271 v = v.transpose(0, 1)<br>
 → 272 attn_output = F.scaled_dot_product_attention(q, k, v, attention_mask, dropout_p=0.0)<br>
273 attn_output = attn_output.transpose(0, 1)<br>
274 attn_output = attn_output.reshape(seq_length, -1)</p>
<p>OutOfMemoryError: CUDA out of memory. Tried to allocate 65.62 GiB. GPU 0 has a total capacity of 31.86 GiB of which 24.54 GiB is free. Of the allocated memory 6.68 GiB is allocated by PyTorch, and 254.10 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (<a href="https://pytorch.org/docs/stable/notes/cuda.html#environment-variables" class="inline-onebox" rel="noopener nofollow ugc">CUDA semantics — PyTorch 2.6 documentation</a>)</p>
<p>Nvidia-smi after getting error</p>
<p>±----------------------------------------------------------------------------------------+<br>
| NVIDIA-SMI 571.96                 Driver Version: 571.96         CUDA Version: 12.8     |<br>
|-----------------------------------------±-----------------------±---------------------+<br>
| GPU  Name                  Driver-Model | Bus-Id          Disp.A | Volatile Uncorr. ECC |<br>
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |<br>
|                                         |                        |               MIG M. |<br>
|=========================================+========================+======================|<br>
|   0  Tesla V100-PCIE-32GB         TCC   |   00000000:18:00.0 Off |                    0 |<br>
| N/A   61C    P0             46W /  250W |    7497MiB /  32768MiB |      0%      Default |<br>
|                                         |                        |                  N/A |<br>
±----------------------------------------±-----------------------±---------------------+<br>
|   1  Tesla V100-PCIE-32GB         TCC   |   00000000:86:00.0 Off |                    0 |<br>
| N/A   46C    P0             39W /  250W |    2305MiB /  32768MiB |      0%      Default |<br>
|                                         |                        |                  N/A |<br>
±----------------------------------------±-----------------------±---------------------+</p>
<p>±----------------------------------------------------------------------------------------+<br>
| Processes:                                                                              |<br>
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |<br>
|        ID   ID                                                               Usage      |<br>
|=========================================================================================|<br>
|    0   N/A  N/A           23300      C   …naconda3\envs\qwen\python.exe       7482MiB |<br>
|    1   N/A  N/A           23300      C   …naconda3\envs\qwen\python.exe       2294MiB |<br>
±----------------------------------------------------------------------------------------+</p>