Upload folder using huggingface_hub

run_transformers_training.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# coding=utf-8
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 # Basic Python imports
 import os
@@ -12,6 +12,10 @@ import time
 import warnings
 from importlib.util import find_spec
 import multiprocessing
+import torch
+import random
+import numpy as np
+from tqdm import tqdm
 
 # Check hardware capabilities first
 CUDA_AVAILABLE = "CUDA_VISIBLE_DEVICES" in os.environ or os.environ.get("NVIDIA_VISIBLE_DEVICES") != ""

transformers_config.json

@@ -67,7 +67,14 @@
       "offload_params": false
     },
     "ddp_find_unused_parameters": false,
-    "dataloader_num_workers": 2
+    "dataloader_num_workers": 2,
+    "ddp_config": {
+      "enabled": true,
+      "backend": "nccl",
+      "find_unused_parameters": false,
+      "broadcast_buffers": false,
+      "gradient_as_bucket_view": true
+    }
   },
 
   "logging": {
@@ -112,11 +119,11 @@
         "use_gradient_checkpointing": true,
         "use_flash_attention": true
       },
-      "multi_gpu_strategy": "data_parallel"
+      "multi_gpu_strategy": "ddp"
     },
     "system_settings": {
-      "cuda_memory_fraction": 0.85,
-      "dataloader_num_workers": 2,
+      "cuda_memory_fraction": 0.75,
+      "dataloader_num_workers": 4,
       "dataloader_pin_memory": true
     },
     "memory_breakdown": {