Saving train state of step 500

checkpoint-500-epoch-6/optimizer.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6dfcbb8d9658d1c23fcc4dd9edb3822ac5914e7ac1dbc60ed30e3cbd3b46a41f
+oid sha256:b4320d7a26b7d96a0c047dc0203176c69c66cc2ebf822d3da84e9f65cd71dda9
 size 3652763351

checkpoint-500-epoch-6/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c0bda92d762ceac4074c24935347a016d4b3eae0db01a4859aa671ec66edc49
+oid sha256:e0f6ba977980b21a89cb91f622f81621c81c1a61d8082dcda5b0e182daab1a4d
 size 2588462170

checkpoint-500-epoch-6/random_states_0.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08efa1fa4270ed24b2ddf0858b252a2544edc723bdada91c58f7ff0c17eb5406
+oid sha256:4d5d5be8b16873cb20ec804deed210623d59e4380818972742d65841aa4d14ce
 size 14344

checkpoint-500-epoch-6/scheduler.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fc1d5d5011b8f5d8cf0fccf1801a3bbade375e254a51919d338cccdd54e3ed8
+oid sha256:a6db3feeafb620e767981163694df9114aa7589fa6f6c074e43f210e5e85019c
 size 1000

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "parler-tts/parler_tts_mini_v0.1",
+  "_name_or_path": "/fsx/yoach/tmp/artefacts/training-400M-punctuated-v2/",
   "architectures": [
     "ParlerTTSForConditionalGeneration"
   ],

run.sh

@@ -29,11 +29,11 @@ python run_parler_tts_training.py \
   --gradient_accumulation_steps 4 \
   --gradient_checkpointing true \
   --per_device_train_batch_size 32 \
-  --learning_rate 3e-5 \
+  --learning_rate 8e-5 \
   --adam_beta1 0.9 \
   --adam_beta2 0.99 \
   --weight_decay 0.01 \
-  --warmup_steps 100 \
+  --warmup_steps 250 \
   --logging_steps 2 \
   --freeze_text_encoder true \
   --audio_encoder_per_device_batch_size 4 \