Training in progress, step 19000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d63b3b7c1076ec597dcdcc6541ece450e677504bd6df3bf6b6d0efdff7161ba
+oid sha256:3e038b96faf7382818cc0badadbf52a75b3504be5e84c5740dff060258144b32
 size 3219908024

wandb/run-20231119_105908-nwx0xdmm/files/output.log

@@ -11853,3 +11853,1091 @@ Reading metadata...: 1650it [00:00, 9802.57it/s]
 
 
 
+
+
+
+
+
+
+
+ 30%|███████████████████████████▍                                                               | 18059/60000 [35:30:39<138:35:50, 11.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▍                                                               | 18080/60000 [35:34:40<138:02:45, 11.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▍                                                               | 18100/60000 [35:38:22<131:04:12, 11.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▍                                                               | 18120/60000 [35:42:35<175:02:36, 15.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▌                                                               | 18140/60000 [35:46:21<149:42:24, 12.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 12945.95it/s]                                               | 18155/60000 [35:49:00<119:18:22, 10.26s/it]
+
+
+
+
+ 30%|███████████████████████████▌                                                               | 18159/60000 [35:49:44<123:22:18, 10.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▌                                                               | 18179/60000 [35:53:23<134:03:21, 11.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▌                                                               | 18200/60000 [35:57:17<127:55:12, 11.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▋                                                               | 18220/60000 [36:01:04<133:06:06, 11.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▋                                                               | 18239/60000 [36:04:52<144:58:13, 12.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▋                                                               | 18260/60000 [36:08:48<126:55:06, 10.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▋                                                               | 18280/60000 [36:12:34<127:34:36, 11.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▊                                                               | 18300/60000 [36:16:40<134:17:22, 11.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▊                                                               | 18319/60000 [36:20:12<137:56:38, 11.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▊                                                               | 18340/60000 [36:24:08<132:23:31, 11.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▊                                                               | 18360/60000 [36:28:14<138:47:15, 12.00s/it]
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▊                                                               | 18369/60000 [36:29:52<120:34:08, 10.43s/it]
+
+ 31%|███████████████████████████▊                                                               | 18370/60000 [36:29:59<108:12:54,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▉                                                               | 18380/60000 [36:31:47<126:05:14, 10.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▉                                                               | 18400/60000 [36:35:23<123:26:11, 10.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▉                                                               | 18420/60000 [36:39:13<123:42:02, 10.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▉                                                               | 18440/60000 [36:43:08<144:32:51, 12.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███████████████████████████▉                                                               | 18460/60000 [36:46:49<125:39:24, 10.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████                                                               | 18480/60000 [36:50:19<118:54:04, 10.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████                                                               | 18499/60000 [36:53:43<122:38:08, 10.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████                                                               | 18519/60000 [36:57:08<118:05:34, 10.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████                                                               | 18539/60000 [37:00:37<116:38:47, 10.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▏                                                              | 18559/60000 [37:03:55<110:08:46,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▏                                                              | 18579/60000 [37:07:20<111:57:28,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▏                                                              | 18599/60000 [37:10:39<122:47:59, 10.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▏                                                              | 18619/60000 [37:14:05<118:02:03, 10.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▎                                                              | 18639/60000 [37:17:47<126:07:31, 10.98s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▎                                                              | 18659/60000 [37:21:06<110:10:44,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▎                                                              | 18679/60000 [37:24:31<108:28:07,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▎                                                              | 18699/60000 [37:27:43<108:56:44,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▍                                                              | 18719/60000 [37:30:59<110:03:06,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▍                                                              | 18739/60000 [37:34:19<115:54:48, 10.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▍                                                              | 18759/60000 [37:37:56<165:45:33, 14.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▍                                                              | 18779/60000 [37:41:10<112:19:00,  9.81s/it]
+
+Reading metadata...: 1650it [00:00, 4113.64it/s]                                                | 18780/60000 [37:41:20<112:18:00,  9.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▌                                                              | 18799/60000 [37:44:21<107:05:38,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▌                                                              | 18819/60000 [37:47:34<111:51:52,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▌                                                              | 18839/60000 [37:50:44<108:03:53,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▌                                                              | 18860/60000 [37:54:07<106:58:49,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▋                                                              | 18880/60000 [37:57:14<106:47:22,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|████████████████████████████▋                                                              | 18899/60000 [38:00:10<105:54:15,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▋                                                              | 18920/60000 [38:03:26<105:27:14,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▋                                                              | 18939/60000 [38:06:27<110:03:11,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▊                                                              | 18959/60000 [38:09:43<110:52:39,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▊                                                              | 18980/60000 [38:13:00<109:19:27,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▊                                                              | 19000/60000 [38:16:35<111:00:17,  9.75s/it][INFO|trainer.py:3173] 2023-11-21 01:15:49,608 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 01:15:49,609 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 01:15:49,609 >>   Batch size = 4
+{'loss': 0.0456, 'learning_rate': 2.0862203389830507e-06, 'epoch': 0.32}
+Reading metadata...: 1704it [00:00, 6722.89it/s]
+[INFO|trainer_utils.py:759] 2023-11-21 01:15:50,677 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.17822265625, 'eval_wer': 8.518658122879758, 'eval_runtime': 591.7997, 'eval_samples_per_second': 2.879, 'eval_steps_per_second': 0.72, 'epoch': 0.32}
+ 32%|████████████████████████████▊                                                              | 19000/60000 [38:26:26<111:00:17,  9.75s/it][INFO|trainer.py:2896] 2023-11-21 01:26:10,667 >> Saving model checkpoint to ./checkpoint-19000
+[INFO|configuration_utils.py:462] 2023-11-21 01:26:10,674 >> Configuration saved in ./checkpoint-19000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 01:26:10,679 >> Configuration saved in ./checkpoint-19000/generation_config.json
+[2023-11-21 01:26:51,330] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step19000 is about to be saved!
+[2023-11-21 01:26:51,354] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-19000/global_step19000/mp_rank_00_model_states.pt
+[2023-11-21 01:26:51,354] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-19000/global_step19000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-21 01:26:51,298 >> Model weights saved in ./checkpoint-19000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 01:26:51,303 >> Feature extractor saved in ./checkpoint-19000/preprocessor_config.json
+[2023-11-21 01:26:56,815] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-19000/global_step19000/mp_rank_00_model_states.pt.
+[2023-11-21 01:26:56,821] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-19000/global_step19000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 01:27:16,145] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-19000/global_step19000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 01:27:16,155] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-19000/global_step19000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 01:27:16,156] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step19000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 01:28:14,214 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▊                                                              | 19019/60000 [38:32:04<113:16:16,  9.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▉                                                              | 19040/60000 [38:35:58<107:24:44,  9.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▉                                                              | 19059/60000 [38:38:59<106:57:07,  9.40s/it]
+
+
+
+
+
+
+
+

wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.1355, "train/learning_rate": 2.1349322033898305e-06, "train/epoch": 0.3, "train/global_step": 18040, "_timestamp": 1700511965.7339036, "_runtime": 127616.6998167038, "_step": 562, "eval/loss": 0.1827392578125, "eval/wer": 8.678854127402941, "eval/runtime": 734.6885, "eval/samples_per_second": 2.319, "eval/steps_per_second": 0.58}
+{"train/loss": 0.0749, "train/learning_rate": 2.0831694915254236e-06, "train/epoch": 0.32, "train/global_step": 19060, "_timestamp": 1700523503.6049488, "_runtime": 139154.5708618164, "_step": 614, "eval/loss": 0.17822265625, "eval/wer": 8.518658122879758, "eval/runtime": 591.7997, "eval/samples_per_second": 2.879, "eval/steps_per_second": 0.72}

wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c661509eff524fa6fc61166ff8fb70b3921709b9feb3fff3ae66325b5480e695
-size 8003044
+oid sha256:bb75bd1ca21ebda7f7845cb9f0501e78a6e6ac1408a014fd785762f2b48b8b87
+size 8749665