Model save

README.md

@@ -35,12 +35,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.001
-- train_batch_size: 4
+- train_batch_size: 2
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- gradient_accumulation_steps: 16
-- total_train_batch_size: 64
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 100

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.99,
-    "train_loss": 3.364711216517857,
-    "train_runtime": 13678.922,
+    "epoch": 3.94,
+    "train_loss": 3.1379870364540503,
+    "train_runtime": 12199.309,
     "train_samples": 2471,
-    "train_samples_per_second": 0.723,
-    "train_steps_per_second": 0.023
+    "train_samples_per_second": 0.81,
+    "train_steps_per_second": 0.012
 }

breeze-listen-w2v2-kn-GF.log

@@ -1,5 +1,5 @@
-02/04/2024 18:56:48 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, 16-bits training: True
-02/04/2024 18:56:48 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+02/04/2024 23:18:42 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, 16-bits training: True
+02/04/2024 23:18:42 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
 _n_gpu=1,
 adafactor=False,
 adam_beta1=0.9,
@@ -39,7 +39,7 @@ fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},
 fsdp_min_num_params=0,
 fsdp_transformer_layer_cls_to_wrap=None,
 full_determinism=False,
-gradient_accumulation_steps=16,
+gradient_accumulation_steps=8,
 gradient_checkpointing=True,
 gradient_checkpointing_kwargs=None,
 greater_is_better=None,
@@ -64,7 +64,7 @@ local_rank=0,
 log_level=passive,
 log_level_replica=warning,
 log_on_each_node=True,
-logging_dir=/cosmos/home/sp-operator/ai/training/models/simpragma/breeze-listen-w2v2-kn-GF/runs/Feb04_18-56-48_knight,
+logging_dir=/cosmos/home/sp-operator/ai/training/models/simpragma/breeze-listen-w2v2-kn-GF/runs/Feb04_23-18-41_knight,
 logging_first_step=False,
 logging_nan_inf_filter=True,
 logging_steps=500,
@@ -84,7 +84,7 @@ output_dir=/cosmos/home/sp-operator/ai/training/models/simpragma/breeze-listen-w
 overwrite_output_dir=True,
 past_index=-1,
 per_device_eval_batch_size=8,
-per_device_train_batch_size=4,
+per_device_train_batch_size=2,
 prediction_loss_only=False,
 push_to_hub=True,
 push_to_hub_model_id=None,
@@ -119,4 +119,5 @@ warmup_ratio=0.0,
 warmup_steps=100,
 weight_decay=0.0,
 )
-{'train_runtime': 12199.309, 'train_samples_per_second': 0.81, 'train_steps_per_second': 0.012, 'train_loss': 3.1379870364540503, 'epoch': 3.94}
+{'loss': 1.1689, 'learning_rate': 0.0002306201550387597, 'epoch': 3.24}
+{'train_runtime': 12565.5546, 'train_samples_per_second': 0.787, 'train_steps_per_second': 0.049, 'train_loss': 0.9830725657475459, 'epoch': 3.99}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5019ad31159ea5df95db8779bd8072d6f2dfcaa73fce699f4d1ef7fdd5b3043
+oid sha256:ba57738ed6370499d692e4056bc3cf252c2f39cd7e074d05d585a03caace6e53
 size 3859264976

train-ctc-model.sh

@@ -44,7 +44,8 @@ export "WORLD_SIZE"="1"
 MODEL=w2v2
 
 # Model names and other stuff
-BASE_MODEL="facebook/mms-1b-all"
+#BASE_MODEL="facebook/mms-1b-all"
+BASE_MODEL="facebook/mms-1b-fl102"
 
 JUST_LANG=${LANG%%_*}
 MY_MODEL="breeze-listen-${MODEL}-${JUST_LANG}-GF"
@@ -55,8 +56,8 @@ echo "OUTDIR: ${OUTDIR}"
 # Training parameters you can tweak. Feel free to directly change any of the parameters below.
 
 MAX_EPOCHS=4
-TRAIN_BATCH_SIZE=1
-EVAL_BATCH_SIZE=1
+TRAIN_BATCH_SIZE=4
+EVAL_BATCH_SIZE=4
 LEARNING_RATE="1e-3"
 
 EVAL_STEPS="1000"
@@ -80,7 +81,7 @@ python ${SCRIPT_DIR}/run_speech_recognition_ctc_adapter.py \
 	--output_dir="${OUTDIR}" \
 	--num_train_epochs="${MAX_EPOCHS}" \
 	--per_device_train_batch_size="${TRAIN_BATCH_SIZE}" \
-	--gradient_accumulation_steps="32"	\
+	--gradient_accumulation_steps="16"	\
 	--learning_rate="${LEARNING_RATE}" \
 	--warmup_steps="100" \
 	--evaluation_strategy="steps" \

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.99,
-    "train_loss": 3.364711216517857,
-    "train_runtime": 13678.922,
+    "epoch": 3.94,
+    "train_loss": 3.1379870364540503,
+    "train_runtime": 12199.309,
     "train_samples": 2471,
-    "train_samples_per_second": 0.723,
-    "train_steps_per_second": 0.023
+    "train_samples_per_second": 0.81,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -1,30 +1,30 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.9886685552407934,
+  "epoch": 3.9352750809061487,
   "eval_steps": 1000,
-  "global_step": 308,
+  "global_step": 152,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 3.99,
-      "step": 308,
-      "total_flos": 1.1188723782736937e+19,
-      "train_loss": 3.364711216517857,
-      "train_runtime": 13678.922,
-      "train_samples_per_second": 0.723,
-      "train_steps_per_second": 0.023
+      "epoch": 3.94,
+      "step": 152,
+      "total_flos": 1.1080848497912578e+19,
+      "train_loss": 3.1379870364540503,
+      "train_runtime": 12199.309,
+      "train_samples_per_second": 0.81,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 500,
-  "max_steps": 308,
+  "max_steps": 152,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 4,
   "save_steps": 1000,
-  "total_flos": 1.1188723782736937e+19,
-  "train_batch_size": 1,
+  "total_flos": 1.1080848497912578e+19,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4aaa9085f71e66c7740edfe87fb1bf5ecbe420672688ea0bc427245f20a7e66a
+oid sha256:0a8b420468056bb42f40608b089d2f99c12e63a02141099c5c66c2c659044452
 size 4856