Training in progress, step 500

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f955048c59e9e4047ed09b4c47e04f34abef271130f0124d9eb58bc873f91c55
+oid sha256:428cd78ecc6d28a896b4ae26c7e9c5a235d077842b48747a6078f9440175197f
 size 8637973773

run_bnb.sh

@@ -6,9 +6,9 @@ python run_speech_recognition_ctc_bnb.py \
 --output_dir="./" \
 --overwrite_output_dir \
 --num_train_epochs="150" \
---per_device_train_batch_size="1" \
---per_device_eval_batch_size="1" \
---gradient_accumulation_steps="48" \
+--per_device_train_batch_size="4" \
+--per_device_eval_batch_size="4" \
+--gradient_accumulation_steps="12" \
 --learning_rate="1e-6" \
 --warmup_steps="2000" \
 --length_column_name="input_length" \

run_speech_recognition_ctc_bnb.py

@@ -698,6 +698,7 @@ def main():
     ]
     optimizer = bnb.optim.Adam8bit(
         params=optimizer_grouped_parameters,
+        lr=training_args.learning_rate,
         betas=(training_args.adam_beta1, training_args.adam_beta2),
         eps=training_args.adam_epsilon,
     )

runs/Jan31_19-04-20_job-2043ddf5-d229-46f3-8808-c26e497d509b/1643656007.0408006/events.out.tfevents.1643656007.job-2043ddf5-d229-46f3-8808-c26e497d509b.574517.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19b17726d5fd4eb996777d0864421aa52c391a515c7c64bf66a7f55400d56ed9
+size 4753

runs/Jan31_19-04-20_job-2043ddf5-d229-46f3-8808-c26e497d509b/events.out.tfevents.1643656007.job-2043ddf5-d229-46f3-8808-c26e497d509b.574517.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d2f453b108916e58e986333948e8aa11506ee8856f1a7e124a8105aafd2d85f
+size 4680

runs/Jan31_19-09-35_job-2043ddf5-d229-46f3-8808-c26e497d509b/1643656313.5386918/events.out.tfevents.1643656313.job-2043ddf5-d229-46f3-8808-c26e497d509b.576732.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149977fd83e8d846363014c92f04f2ce03392e3cf21087ec2bdc24264f2c253d
+size 4753

runs/Jan31_19-09-35_job-2043ddf5-d229-46f3-8808-c26e497d509b/events.out.tfevents.1643656313.job-2043ddf5-d229-46f3-8808-c26e497d509b.576732.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c544cac7886eaa2ef2904ae9cf0a94e3f6b85d40af8c0153c15bdeb4c52cb28
+size 12812

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c3eb05246c3b8f398233525187f3e1378df4480944773dffe7ed37d96ccc4e4b
-size 3055
+oid sha256:a8d2f4f0bcd677abf1431f17fe99eb16ecf98fc5d369730ce6ec691c318a1c27
+size 2991