diff --git "a/wandb/run-20231119_105908-nwx0xdmm/files/output.log" "b/wandb/run-20231119_105908-nwx0xdmm/files/output.log"
--- "a/wandb/run-20231119_105908-nwx0xdmm/files/output.log"
+++ "b/wandb/run-20231119_105908-nwx0xdmm/files/output.log"
@@ -14016,3 +14016,6880 @@ Reading metadata...: 1704it [00:00, 8993.55it/s]
 
 
 
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▍                                                            | 20079/60000 [41:36:10<104:43:13,  9.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▍                                                            | 20099/60000 [41:39:21<102:28:57,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▌                                                            | 20120/60000 [41:42:39<103:10:01,  9.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▌                                                            | 20139/60000 [41:45:36<103:42:49,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▌                                                            | 20160/60000 [41:48:52<104:50:00,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▌                                                            | 20180/60000 [41:52:03<103:59:30,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▋                                                            | 20200/60000 [41:55:09<103:43:33,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▋                                                            | 20220/60000 [41:58:15<103:08:21,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▋                                                            | 20239/60000 [42:01:14<104:16:36,  9.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▋                                                            | 20260/60000 [42:04:34<110:43:14, 10.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▊                                                            | 20280/60000 [42:07:56<103:40:30,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▊                                                            | 20300/60000 [42:11:16<101:47:45,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▊                                                            | 20320/60000 [42:14:22<103:26:00,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▊                                                            | 20340/60000 [42:17:27<101:40:12,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▉                                                            | 20360/60000 [42:20:39<103:11:34,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▉                                                            | 20380/60000 [42:23:45<100:46:47,  9.16s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▉                                                            | 20400/60000 [42:26:52<102:08:45,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|██████████████████████████████▉                                                            | 20420/60000 [42:29:58<102:05:41,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████                                                            | 20440/60000 [42:33:11<103:48:22,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████                                                            | 20460/60000 [42:36:17<102:03:14,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████                                                            | 20480/60000 [42:39:27<101:25:33,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████                                                            | 20500/60000 [42:42:33<101:05:33,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████                                                            | 20520/60000 [42:45:39<102:21:48,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▏                                                           | 20540/60000 [42:48:50<101:24:36,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▏                                                           | 20560/60000 [42:52:20<108:37:50,  9.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▏                                                           | 20579/60000 [42:55:16<101:06:12,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▏                                                           | 20599/60000 [42:58:23<102:52:01,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▎                                                           | 20619/60000 [43:01:45<126:00:02, 11.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▎                                                           | 20639/60000 [43:04:51<100:52:25,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▎                                                           | 20659/60000 [43:07:57<101:39:57,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▎                                                           | 20679/60000 [43:11:03<101:38:08,  9.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███████████████████████████████▍                                                           | 20699/60000 [43:14:13<117:10:49, 10.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▍                                                           | 20719/60000 [43:17:18<100:30:41,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▍                                                           | 20739/60000 [43:20:25<101:21:26,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 13484.43it/s]                                               | 20754/60000 [43:22:44<101:05:28,  9.27s/it]
+
+
+
+
+
+ 35%|███████████████████████████████▍                                                           | 20759/60000 [43:23:32<101:59:08,  9.36s/it]
+
+Reading metadata...: 1650it [00:00, 10321.66it/s]                                               | 20760/60000 [43:23:41<101:22:53,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▌                                                           | 20779/60000 [43:26:38<101:18:29,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▌                                                           | 20800/60000 [43:29:59<102:07:43,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▌                                                           | 20819/60000 [43:32:55<100:25:02,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▌                                                           | 20839/60000 [43:36:00<100:47:45,  9.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▋                                                           | 20859/60000 [43:39:38<100:04:39,  9.20s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████��██████████████████████████▋                                                           | 20880/60000 [43:42:58<102:52:32,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▋                                                           | 20899/60000 [43:46:12<107:46:48,  9.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▋                                                           | 20919/60000 [43:49:18<100:27:49,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▊                                                           | 20940/60000 [43:52:35<101:49:30,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▊                                                           | 20959/60000 [43:55:41<102:31:18,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▊                                                           | 20980/60000 [43:59:01<101:29:01,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▊                                                           | 20999/60000 [44:01:58<101:28:52,  9.37s/it]
+ 35%|███████████████████████████████▊                                                           | 21000/60000 [44:02:08<101:54:19,  9.41s/it][INFO|trainer.py:3173] 2023-11-21 07:01:22,922 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 07:01:22,922 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 07:01:22,922 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 4141.28it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-21 07:01:24,721 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+ 35%|███████████████████████████████▊                                                           | 21000/60000 [44:12:10<101:54:19,  9.41s/it]
+ 35%|███████████████████████████████▊                                                           | 21000/60000 [44:12:10<101:54:19,  9.41s/it][INFO|trainer.py:2896] 2023-11-21 07:11:37,065 >> Saving model checkpoint to ./checkpoint-21000
+[INFO|configuration_utils.py:462] 2023-11-21 07:11:37,071 >> Configuration saved in ./checkpoint-21000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 07:11:37,086 >> Configuration saved in ./checkpoint-21000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-21 07:11:59,193 >> Model weights saved in ./checkpoint-21000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 07:11:59,199 >> Feature extractor saved in ./checkpoint-21000/preprocessor_config.json
+[2023-11-21 07:11:59,218] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step21000 is about to be saved!
+[2023-11-21 07:11:59,300] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-21000/global_step21000/mp_rank_00_model_states.pt
+[2023-11-21 07:11:59,300] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-21000/global_step21000/mp_rank_00_model_states.pt...
+[2023-11-21 07:12:08,069] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-21000/global_step21000/mp_rank_00_model_states.pt.
+[2023-11-21 07:12:08,079] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-21000/global_step21000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 07:12:29,737] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-21000/global_step21000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 07:12:29,745] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-21000/global_step21000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 07:12:29,746] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step21000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 07:13:25,889 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▉                                                           | 21019/60000 [44:17:24<112:02:41, 10.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▉                                                           | 21039/60000 [44:20:32<101:29:50,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▉                                                           | 21060/60000 [44:23:57<102:19:47,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███████████████████████████████▉                                                           | 21080/60000 [44:27:17<103:03:05,  9.53s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████                                                           | 21100/60000 [44:30:27<102:13:26,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████                                                           | 21119/60000 [44:33:27<103:06:30,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████                                                           | 21139/60000 [44:36:39<116:56:17, 10.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████                                                           | 21160/60000 [44:39:56<100:47:25,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████                                                           | 21179/60000 [44:42:54<100:33:07,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████▌                                                           | 21200/60000 [44:46:10<99:36:18,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████▌                                                           | 21219/60000 [44:49:06<98:52:38,  9.18s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████▏                                                          | 21239/60000 [44:52:18<101:56:07,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████▌                                                           | 21259/60000 [44:55:23<99:54:35,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|████████████████████████████████▋                                                           | 21280/60000 [44:58:37<99:26:38,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▎                                                          | 21300/60000 [45:02:15<102:33:30,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▎                                                          | 21319/60000 [45:05:16<106:34:57,  9.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▋                                                           | 21340/60000 [45:08:31<98:39:19,  9.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▍                                                          | 21359/60000 [45:11:28<100:12:46,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▍                                                          | 21380/60000 [45:15:24<132:22:59, 12.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▍                                                          | 21400/60000 [45:18:36<100:26:28,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▍                                                          | 21420/60000 [45:21:49<100:55:45,  9.42s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███��████████████████████████████▌                                                          | 21439/60000 [45:24:48<101:35:01,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▌                                                          | 21460/60000 [45:28:15<105:57:54,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▌                                                          | 21480/60000 [45:31:33<106:30:07,  9.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▌                                                          | 21500/60000 [45:34:54<107:02:52, 10.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▉                                                           | 21519/60000 [45:37:54<99:40:32,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████                                                           | 21539/60000 [45:40:59<98:23:17,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████                                                           | 21559/60000 [45:44:07<99:44:53,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▋                                                          | 21579/60000 [45:47:13<100:00:18,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▊                                                          | 21600/60000 [45:50:35<100:22:53,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▊                                                          | 21620/60000 [45:53:51<112:57:52, 10.60s/it]
+
+
+
+ 36%|█████████████████████████████████▏                                                          | 21623/60000 [45:54:16<93:17:22,  8.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▏                                                          | 21640/60000 [45:56:53<98:50:19,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▏                                                          | 21660/60000 [46:00:00<98:34:35,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|████████████████████████████████▉                                                          | 21680/60000 [46:03:14<102:20:24,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▎                                                          | 21700/60000 [46:06:21<99:49:14,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▎                                                          | 21720/60000 [46:09:26<98:46:04,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▎                                                          | 21740/60000 [46:12:33<99:50:42,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 7772.12it/s]                                                | 21750/60000 [46:14:26<141:25:59, 13.31s/it]
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████                                                          | 21760/60000 [46:16:04<104:50:00,  9.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████                                                          | 21780/60000 [46:19:19<100:52:25,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▍                                                          | 21799/60000 [46:22:18<99:26:29,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███████████���█████████████████████                                                          | 21819/60000 [46:25:27<101:29:49,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▍                                                          | 21840/60000 [46:28:43<97:58:22,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▏                                                         | 21860/60000 [46:32:18<108:28:36, 10.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▌                                                          | 21879/60000 [46:35:15<98:36:15,  9.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|█████████████████████████████████▏                                                         | 21899/60000 [46:38:24<100:11:02,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▏                                                         | 21919/60000 [46:41:35<101:47:46,  9.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▎                                                         | 21939/60000 [46:44:52<104:12:16,  9.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▎                                                         | 21959/60000 [46:48:08<100:00:40,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▋                                                          | 21980/60000 [46:51:23<98:55:55,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▋                                                          | 21999/60000 [46:54:20<99:12:52,  9.40s/it]
+ 37%|█████████████████████████████████▎                                                         | 22000/60000 [46:54:30<100:45:44,  9.55s/it][INFO|trainer.py:3173] 2023-11-21 09:53:45,184 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 09:53:45,185 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 09:53:45,185 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 8936.45it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-21 09:53:46,128 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.1820068359375, 'eval_wer': 8.528081417263476, 'eval_runtime': 601.4562, 'eval_samples_per_second': 2.833, 'eval_steps_per_second': 0.708, 'epoch': 0.37}
+ 37%|█████████████████████████████████▎                                                         | 22000/60000 [47:04:32<100:45:44,  9.55s/it][INFO|trainer.py:2896] 2023-11-21 10:04:17,934 >> Saving model checkpoint to ./checkpoint-22000
+[INFO|configuration_utils.py:462] 2023-11-21 10:04:17,944 >> Configuration saved in ./checkpoint-22000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 10:04:17,948 >> Configuration saved in ./checkpoint-22000/generation_config.json
+[2023-11-21 10:05:05,458] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step22000 is about to be saved!
+[2023-11-21 10:05:05,479] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-22000/global_step22000/mp_rank_00_model_states.pt
+[2023-11-21 10:05:05,479] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-22000/global_step22000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-21 10:05:05,440 >> Model weights saved in ./checkpoint-22000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 10:05:05,445 >> Feature extractor saved in ./checkpoint-22000/preprocessor_config.json
+[2023-11-21 10:05:15,298] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-22000/global_step22000/mp_rank_00_model_states.pt.
+[2023-11-21 10:05:15,316] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-22000/global_step22000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 10:05:40,030] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-22000/global_step22000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 10:05:40,041] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-22000/global_step22000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 10:05:40,042] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step22000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 10:06:38,208 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▍                                                         | 22019/60000 [47:10:27<104:40:14,  9.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▍                                                         | 22039/60000 [47:14:12<101:51:28,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 10884.72it/s]                                               | 22053/60000 [47:16:25<100:24:43,  9.53s/it]
+
+
+
+
+
+
+ 37%|█████████████████████████████████▍                                                         | 22059/60000 [47:17:24<101:43:10,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▍                                                         | 22079/60000 [47:20:40<101:41:23,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▉                                                          | 22099/60000 [47:23:49<98:05:38,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▌                                                         | 22119/60000 [47:27:03<100:20:53,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▉                                                          | 22139/60000 [47:30:10<98:54:48,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▉                                                          | 22160/60000 [47:33:29<99:27:33,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████                                                          | 22179/60000 [47:36:28<97:32:43,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|█████████████████████████████████▋                                                         | 22199/60000 [47:39:39<107:46:06, 10.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████                                                          | 22219/60000 [47:42:46<98:15:45,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████                                                          | 22239/60000 [47:45:54<99:18:55,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▏                                                         | 22259/60000 [47:49:00<97:02:42,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▏                                                         | 22279/60000 [47:52:09<98:56:59,  9.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▏                                                         | 22300/60000 [47:55:40<97:28:49,  9.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▏                                                         | 22319/60000 [47:58:37<97:27:39,  9.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▎                                                         | 22339/60000 [48:01:43<98:52:52,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███████████████████████���██████████▎                                                         | 22359/60000 [48:04:50<98:06:38,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▎                                                         | 22379/60000 [48:08:00<99:53:28,  9.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▎                                                         | 22399/60000 [48:11:08<97:22:27,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▍                                                         | 22420/60000 [48:14:25<98:12:37,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▍                                                         | 22439/60000 [48:17:22<96:07:56,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▍                                                         | 22459/60000 [48:20:28<96:53:35,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|██████████████████████████████████▍                                                         | 22480/60000 [48:23:47<95:10:57,  9.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                         | 22500/60000 [48:26:53<95:56:35,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▏                                                        | 22520/60000 [48:30:11<109:28:18, 10.51s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                         | 22540/60000 [48:33:18<96:13:10,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                         | 22559/60000 [48:36:20<97:07:08,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                         | 22580/60000 [48:39:35<96:44:00,  9.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▋                                                         | 22599/60000 [48:42:32<96:33:17,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▎                                                        | 22620/60000 [48:45:59<106:36:32, 10.27s/it]
+
+
+
+
+ 38%|██████████████████████████████████▋                                                         | 22624/60000 [48:46:33<88:54:09,  8.56s/it]
+
+ 38%|██████████████████████████████████▋                                                         | 22625/60000 [48:46:39<81:48:56,  7.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▋                                                         | 22639/60000 [48:48:50<98:20:02,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▎                                                        | 22660/60000 [48:52:37<126:14:58, 12.17s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▊                                                         | 22680/60000 [48:55:43<95:38:07,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▊                                                         | 22700/60000 [48:58:57<99:44:08,  9.63s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▍                                                        | 22720/60000 [49:02:08<100:35:44,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▊                                                         | 22740/60000 [49:05:20<95:18:31,  9.21s/it]
+Reading metadata...: 1650it [00:00, 7451.68it/s]                                                 | 22741/60000 [49:05:29<95:37:07,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                        | 22759/60000 [49:08:43<147:57:14, 14.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▉                                                         | 22780/60000 [49:12:00<96:03:03,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                        | 22800/60000 [49:15:22<103:32:34, 10.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▌                                                        | 22819/60000 [49:18:25<102:39:58,  9.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████                                                         | 22840/60000 [49:21:41<96:49:33,  9.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████                                                         | 22860/60000 [49:24:50<95:53:40,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████                                                         | 22879/60000 [49:27:50<98:57:57,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████                                                         | 22899/60000 [49:30:57<97:39:17,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▏                                                        | 22920/60000 [49:34:18<96:48:01,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▏                                                        | 22940/60000 [49:37:24<94:58:13,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▏                                                        | 22959/60000 [49:40:30<96:04:32,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▏                                                        | 22980/60000 [49:43:46<94:46:28,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▎                                                        | 23000/60000 [49:46:58<95:46:43,  9.32s/it][INFO|trainer.py:3173] 2023-11-21 12:46:13,232 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 12:46:13,232 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 12:46:13,232 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 8490.40it/s]
+[INFO|trainer_utils.py:759] 2023-11-21 12:46:14,237 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'loss': 0.0463, 'learning_rate': 1.883084745762712e-06, 'epoch': 0.38}
+ 38%|███████████████████████████████████▎                                                        | 23000/60000 [49:57:06<95:46:43,  9.32s/it]
+ 38%|███████████████████████████████████▎                                                        | 23000/60000 [49:57:06<95:46:43,  9.32s/it][INFO|trainer.py:2896] 2023-11-21 12:56:42,686 >> Saving model checkpoint to ./checkpoint-23000
+[INFO|configuration_utils.py:462] 2023-11-21 12:56:42,697 >> Configuration saved in ./checkpoint-23000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 12:56:42,710 >> Configuration saved in ./checkpoint-23000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-21 12:57:28,606 >> Model weights saved in ./checkpoint-23000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 12:57:28,613 >> Feature extractor saved in ./checkpoint-23000/preprocessor_config.json
+[2023-11-21 12:57:28,646] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step23000 is about to be saved!
+[2023-11-21 12:57:28,731] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-23000/global_step23000/mp_rank_00_model_states.pt
+[2023-11-21 12:57:28,732] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-23000/global_step23000/mp_rank_00_model_states.pt...
+[2023-11-21 12:57:37,797] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-23000/global_step23000/mp_rank_00_model_states.pt.
+[2023-11-21 12:57:37,803] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-23000/global_step23000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 12:57:57,138] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-23000/global_step23000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 12:57:57,147] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-23000/global_step23000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 12:57:57,148] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step23000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 12:58:55,172 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|██████████████████████████████████▉                                                        | 23020/60000 [50:02:54<100:26:12,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▎                                                        | 23040/60000 [50:06:04<99:14:41,  9.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▎                                                        | 23060/60000 [50:09:15<97:21:26,  9.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████                                                        | 23080/60000 [50:12:33<113:46:18, 11.09s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███████████████████████████████████▍                                                        | 23100/60000 [50:15:41<95:58:13,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▍                                                        | 23119/60000 [50:18:39<95:31:48,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▍                                                        | 23140/60000 [50:21:54<95:03:51,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▏                                                       | 23160/60000 [50:25:39<215:56:02, 21.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▌                                                        | 23180/60000 [50:28:51<95:16:19,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▌                                                        | 23200/60000 [50:32:00<96:32:42,  9.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▌                                                        | 23220/60000 [50:35:04<94:06:45,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▋                                                        | 23240/60000 [50:38:10<93:37:16,  9.17s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▎                                                       | 23260/60000 [50:41:21<101:23:37,  9.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|██��████████████████████████████████▋                                                        | 23279/60000 [50:44:33<97:17:24,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▋                                                        | 23300/60000 [50:48:03<97:17:24,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▊                                                        | 23320/60000 [50:51:11<96:22:56,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▊                                                        | 23339/60000 [50:54:09<96:14:37,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 12657.98it/s]                                                | 23353/60000 [50:56:25<94:59:11,  9.33s/it]
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▊                                                        | 23360/60000 [50:57:31<94:54:32,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▊                                                        | 23379/60000 [51:00:28<94:12:49,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▉                                                        | 23400/60000 [51:03:42<94:22:16,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▉                                                        | 23420/60000 [51:06:49<95:00:41,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▉                                                        | 23440/60000 [51:10:02<96:06:33,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▉                                                        | 23460/60000 [51:13:08<93:39:07,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████                                                        | 23480/60000 [51:16:15<94:32:40,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███████████████████████████████████▋                                                       | 23500/60000 [51:19:46<100:41:14,  9.93s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████                                                        | 23520/60000 [51:22:54<95:27:04,  9.42s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████                                                        | 23540/60000 [51:26:07<93:19:50,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▏                                                       | 23560/60000 [51:29:13<93:31:32,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▏                                                       | 23580/60000 [51:32:45<95:58:45,  9.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▏                                                       | 23600/60000 [51:35:50<92:28:39,  9.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▏                                                       | 23620/60000 [51:39:11<99:17:18,  9.83s/it]
+
+
+
+
+
+
+ 39%|████████████████████████████████████▏                                                       | 23626/60000 [51:40:03<84:43:43,  8.39s/it]
+[2023-11-21 14:39:18,403] [INFO] [loss_scaler.py:190:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 131072, but hysteresis is 2. Reducing hysteresis to 1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▏                                                       | 23640/60000 [51:42:11<94:07:28,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▎                                                       | 23660/60000 [51:45:15<93:01:22,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|████████████████████████████████████▎                                                       | 23680/60000 [51:48:21<93:24:48,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|███████████████████████████████████▉                                                       | 23700/60000 [51:51:32<101:50:55, 10.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▎                                                       | 23719/60000 [51:54:29<94:03:13,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 10372.32it/s]                                                | 23731/60000 [51:56:21<94:49:48,  9.41s/it]
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▍                                                       | 23739/60000 [51:57:38<94:36:59,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▍                                                       | 23759/60000 [52:01:03<96:08:41,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▍                                                       | 23779/60000 [52:04:10<93:29:28,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▍                                                       | 23799/60000 [52:07:23<93:12:16,  9.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▌                                                       | 23819/60000 [52:10:31<93:28:15,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▌                                                       | 23839/60000 [52:13:37<95:52:23,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▌                                                       | 23860/60000 [52:16:56<95:24:39,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▌                                                       | 23879/60000 [52:19:58<95:55:45,  9.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▋                                                       | 23899/60000 [52:23:05<93:44:10,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▋                                                       | 23919/60000 [52:26:11<93:00:43,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▋                                                       | 23940/60000 [52:29:29<95:56:20,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▋                                                       | 23959/60000 [52:32:36<94:03:17,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▊                                                       | 23979/60000 [52:35:49<95:30:53,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▊                                                       | 23999/60000 [52:38:57<93:25:58,  9.34s/it]
+ 40%|████████████████████████████████████▊                                                       | 24000/60000 [52:39:06<93:16:16,  9.33s/it][INFO|trainer.py:3173] 2023-11-21 15:38:21,410 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 15:38:21,411 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 15:38:21,411 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 8355.48it/s]
+[INFO|trainer_utils.py:759] 2023-11-21 15:38:22,382 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.1917724609375, 'eval_wer': 8.452695062193744, 'eval_runtime': 605.9268, 'eval_samples_per_second': 2.812, 'eval_steps_per_second': 0.703, 'epoch': 0.4}
+ 40%|████████████████████████████████████▊                                                       | 24000/60000 [52:49:12<93:16:16,  9.33s/it][INFO|trainer.py:2896] 2023-11-21 15:48:55,359 >> Saving model checkpoint to ./checkpoint-24000
+[INFO|configuration_utils.py:462] 2023-11-21 15:48:55,366 >> Configuration saved in ./checkpoint-24000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 15:48:55,371 >> Configuration saved in ./checkpoint-24000/generation_config.json
+[2023-11-21 15:49:36,231] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step24000 is about to be saved!
+[2023-11-21 15:49:36,259] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-24000/global_step24000/mp_rank_00_model_states.pt
+[2023-11-21 15:49:36,260] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-24000/global_step24000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-21 15:49:36,197 >> Model weights saved in ./checkpoint-24000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 15:49:36,203 >> Feature extractor saved in ./checkpoint-24000/preprocessor_config.json
+[2023-11-21 15:49:46,079] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-24000/global_step24000/mp_rank_00_model_states.pt.
+[2023-11-21 15:49:46,092] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-24000/global_step24000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 15:50:18,129] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-24000/global_step24000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 15:50:18,137] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-24000/global_step24000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 15:50:18,138] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step24000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 15:51:16,618 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▍                                                      | 24019/60000 [52:55:10<101:14:13, 10.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▊                                                       | 24039/60000 [52:58:35<95:51:40,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▉                                                       | 24059/60000 [53:01:53<96:28:53,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▉                                                       | 24079/60000 [53:05:04<93:45:00,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▉                                                       | 24099/60000 [53:08:11<94:26:12,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▉                                                       | 24120/60000 [53:11:30<94:43:06,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████████████████████████████████████▌                                                      | 24139/60000 [53:14:32<102:08:33, 10.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████                                                       | 24159/60000 [53:17:40<94:18:35,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████                                                       | 24179/60000 [53:20:47<93:24:27,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████                                                       | 24199/60000 [53:23:55<93:35:42,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████▏                                                      | 24219/60000 [53:27:01<93:03:50,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████▏                                                      | 24240/60000 [53:30:50<93:38:02,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████▏                                                      | 24260/60000 [53:33:57<93:01:53,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████▏                                                      | 24280/60000 [53:37:06<93:10:14,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|█████████████████████████████████████▎                                                      | 24300/60000 [53:40:24<94:36:07,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████████████████████████████████████▉                                                      | 24319/60000 [53:43:38<103:24:07, 10.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▎                                                      | 24339/60000 [53:46:52<95:14:58,  9.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▎                                                      | 24359/60000 [53:50:03<95:15:48,  9.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▍                                                      | 24380/60000 [53:53:21<92:25:02,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▍                                                      | 24399/60000 [53:56:21<92:54:26,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▍                                                      | 24420/60000 [53:59:46<93:35:17,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▍                                                      | 24440/60000 [54:02:56<93:23:46,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▌                                                      | 24460/60000 [54:06:06<93:14:41,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▌                                                      | 24480/60000 [54:09:49<96:09:19,  9.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▌                                                      | 24499/60000 [54:12:51<92:48:11,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▌                                                      | 24520/60000 [54:16:07<93:11:53,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▋                                                      | 24540/60000 [54:19:13<90:50:48,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████���███████████████▋                                                      | 24560/60000 [54:22:19<93:38:47,  9.51s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▋                                                      | 24579/60000 [54:25:15<90:47:11,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▋                                                      | 24600/60000 [54:28:36<92:17:27,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▊                                                      | 24620/60000 [54:31:51<96:41:38,  9.84s/it]
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▊                                                      | 24627/60000 [54:32:57<92:17:47,  9.39s/it]
+
+ 41%|█████████████████████████████████████▊                                                      | 24628/60000 [54:33:03<82:53:37,  8.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▊                                                      | 24640/60000 [54:34:52<92:02:30,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 13319.12it/s]                                                | 24652/60000 [54:36:44<90:55:05,  9.26s/it]
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▊                                                      | 24659/60000 [54:37:50<91:45:06,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▊                                                      | 24680/60000 [54:41:10<91:43:57,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▍                                                     | 24699/60000 [54:44:23<115:24:07, 11.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▉                                                      | 24719/60000 [54:47:32<93:00:42,  9.49s/it]
+
+
+Reading metadata...: 1650it [00:00, 2908.17it/s]                                                 | 24721/60000 [54:47:51<93:16:47,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▉                                                      | 24739/60000 [54:50:45<92:33:29,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▉                                                      | 24760/60000 [54:54:07<95:11:39,  9.72s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▉                                                      | 24779/60000 [54:57:14<93:08:34,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▌                                                     | 24799/60000 [55:01:03<104:11:49, 10.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|█████████████████████████████████████▋                                                     | 24820/60000 [55:04:43<103:31:37, 10.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|██████████████████████████████████████                                                      | 24840/60000 [55:08:07<95:13:26,  9.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|██████████████████████████████████████                                                      | 24860/60000 [55:11:22<93:29:32,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|██████████████████████████████████████▏                                                     | 24880/60000 [55:14:37<98:38:23, 10.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████���███████████████████▏                                                     | 24900/60000 [55:17:49<91:21:24,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▏                                                     | 24920/60000 [55:21:03<94:23:00,  9.69s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▏                                                     | 24940/60000 [55:24:19<95:10:24,  9.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▎                                                     | 24960/60000 [55:28:09<90:13:15,  9.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▎                                                     | 24979/60000 [55:31:40<95:02:47,  9.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▎                                                     | 25000/60000 [55:35:01<93:04:09,  9.57s/it][INFO|trainer.py:3173] 2023-11-21 18:34:15,793 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 18:34:15,793 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 18:34:15,793 >>   Batch size = 4
+Reading metadata...: 1it [00:00,  6.16it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-21 18:34:17,113 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.1820068359375, 'eval_wer': 8.311345646437996, 'eval_runtime': 612.2957, 'eval_samples_per_second': 2.783, 'eval_steps_per_second': 0.696, 'epoch': 0.42}
+ 42%|██████████████████████████████████████▎                                                     | 25000/60000 [55:45:13<93:04:09,  9.57s/it][INFO|trainer.py:2896] 2023-11-21 18:45:00,361 >> Saving model checkpoint to ./checkpoint-25000
+[INFO|configuration_utils.py:462] 2023-11-21 18:45:00,373 >> Configuration saved in ./checkpoint-25000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 18:45:00,381 >> Configuration saved in ./checkpoint-25000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-21 18:45:42,429 >> Model weights saved in ./checkpoint-25000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 18:45:42,455 >> Feature extractor saved in ./checkpoint-25000/preprocessor_config.json
+[2023-11-21 18:45:42,860] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step25000 is about to be saved!
+[2023-11-21 18:45:42,898] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-25000/global_step25000/mp_rank_00_model_states.pt
+[2023-11-21 18:45:42,898] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-25000/global_step25000/mp_rank_00_model_states.pt...
+[2023-11-21 18:45:48,554] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-25000/global_step25000/mp_rank_00_model_states.pt.
+[2023-11-21 18:45:48,559] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-25000/global_step25000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 18:46:09,786] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-25000/global_step25000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 18:46:09,795] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-25000/global_step25000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 18:46:09,796] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step25000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 18:47:10,590 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|█████████████████████████████████████▉                                                     | 25019/60000 [55:51:06<100:19:49, 10.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▍                                                     | 25039/60000 [55:54:26<92:29:28,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▍                                                     | 25059/60000 [55:57:46<99:09:24, 10.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▍                                                     | 25079/60000 [56:01:08<94:34:43,  9.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▍                                                     | 25099/60000 [56:04:20<96:22:22,  9.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▌                                                     | 25119/60000 [56:07:42<94:45:09,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▌                                                     | 25139/60000 [56:11:00<95:48:51,  9.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▌                                                     | 25159/60000 [56:14:13<91:31:31,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▌                                                     | 25179/60000 [56:17:29<92:59:29,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▏                                                    | 25200/60000 [56:21:05<101:06:36, 10.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▋                                                     | 25219/60000 [56:24:10<91:26:56,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▋                                                     | 25239/60000 [56:27:29<98:01:52, 10.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▋                                                     | 25259/60000 [56:30:51<99:18:45, 10.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▊                                                     | 25279/60000 [56:34:11<95:59:56,  9.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▊                                                     | 25300/60000 [56:37:56<92:37:56,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▊                                                     | 25319/60000 [56:41:01<94:00:31,  9.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▊                                                     | 25340/60000 [56:44:29<98:27:29, 10.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▉                                                     | 25359/60000 [56:47:40<96:51:19, 10.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▍                                                    | 25379/60000 [56:51:26<145:20:28, 15.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▉                                                     | 25399/60000 [56:54:43<92:04:11,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|██████████████████████████████████████▉                                                     | 25420/60000 [56:58:10<94:13:19,  9.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|███████████████████████████████████████                                                     | 25439/60000 [57:01:17<95:44:06,  9.97s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|███████████████████████████████████████                                                     | 25459/60000 [57:04:35<93:50:31,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|███████████████████████████████████████                                                     | 25479/60000 [57:07:54<91:49:50,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|███████████████████████████████████████                                                     | 25500/60000 [57:11:28<99:11:30, 10.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▏                                                    | 25520/60000 [57:14:55<94:17:46,  9.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▏                                                    | 25540/60000 [57:18:29<95:40:40, 10.00s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▏                                                    | 25560/60000 [57:21:45<90:25:38,  9.45s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▏                                                    | 25579/60000 [57:24:51<95:46:13, 10.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                    | 25600/60000 [57:28:20<95:11:54,  9.96s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                    | 25619/60000 [57:31:31<97:22:21, 10.20s/it]
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                    | 25629/60000 [57:33:06<89:37:29,  9.39s/it]
+
+ 43%|███████████████████████████████████████▎                                                    | 25630/60000 [57:33:13<82:25:03,  8.63s/it]
+
+
+
+
+
+
+
+
+
+ 43%|██████████████████████████████████████▉                                                    | 25639/60000 [57:34:40<103:36:33, 10.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                    | 25659/60000 [57:38:06<93:07:19,  9.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                    | 25679/60000 [57:41:18<91:18:35,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|██████████████████████████████████████▉                                                    | 25699/60000 [57:44:56<162:20:20, 17.04s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 7197.96it/s]                                                 | 25711/60000 [57:46:52<93:40:48,  9.84s/it]
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▍                                                    | 25720/60000 [57:48:19<91:01:44,  9.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▍                                                    | 25740/60000 [57:51:39<93:42:03,  9.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▍                                                    | 25759/60000 [57:54:45<91:10:14,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▌                                                    | 25780/60000 [57:58:11<93:52:05,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▌                                                    | 25799/60000 [58:01:21<97:04:12, 10.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▌                                                    | 25820/60000 [58:04:53<93:29:26,  9.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▌                                                    | 25840/60000 [58:08:08<93:01:06,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▏                                                   | 25859/60000 [58:11:29<106:48:54, 11.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                   | 25880/60000 [58:15:16<100:05:48, 10.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                   | 25900/60000 [58:18:54<103:24:18, 10.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                   | 25920/60000 [58:22:40<102:46:12, 10.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                   | 25940/60000 [58:26:19<102:57:50, 10.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▎                                                   | 25951/60000 [58:28:27<109:01:34, 11.53s/it]
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▊                                                    | 25959/60000 [58:29:51<96:50:12, 10.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▊                                                    | 25980/60000 [58:33:22<95:53:08, 10.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▊                                                    | 26000/60000 [58:36:57<98:33:45, 10.44s/it][INFO|trainer.py:3173] 2023-11-21 21:36:12,243 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 21:36:12,244 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 21:36:12,245 >>   Batch size = 4
+Reading metadata...: 0it [00:00, ?it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-21 21:36:13,298 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.197509765625, 'eval_wer': 8.301922352054278, 'eval_runtime': 649.294, 'eval_samples_per_second': 2.624, 'eval_steps_per_second': 0.656, 'epoch': 0.43}
+ 43%|███████████████████████████████████████▊                                                    | 26000/60000 [58:47:47<98:33:45, 10.44s/it][INFO|trainer.py:2896] 2023-11-21 21:47:29,639 >> Saving model checkpoint to ./checkpoint-26000
+[INFO|configuration_utils.py:462] 2023-11-21 21:47:29,648 >> Configuration saved in ./checkpoint-26000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 21:47:29,654 >> Configuration saved in ./checkpoint-26000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-21 21:48:11,967 >> Model weights saved in ./checkpoint-26000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 21:48:11,976 >> Feature extractor saved in ./checkpoint-26000/preprocessor_config.json
+[2023-11-21 21:48:12,023] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step26000 is about to be saved!
+[2023-11-21 21:48:12,057] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-26000/global_step26000/mp_rank_00_model_states.pt
+[2023-11-21 21:48:12,057] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-26000/global_step26000/mp_rank_00_model_states.pt...
+[2023-11-21 21:48:18,344] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-26000/global_step26000/mp_rank_00_model_states.pt.
+[2023-11-21 21:48:18,353] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-26000/global_step26000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 21:48:43,408] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-26000/global_step26000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 21:48:43,420] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-26000/global_step26000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 21:48:43,421] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step26000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 21:49:50,035 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▉                                                    | 26020/60000 [58:55:11<98:22:34, 10.42s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▉                                                    | 26040/60000 [58:58:35<98:01:10, 10.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████▉                                                    | 26060/60000 [59:02:08<99:53:05, 10.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                                                                                                                           | 26080/60000 [59:05:57<154:00:39, 16.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                                                                                                           | 26100/60000 [59:09:41<102:52:30, 10.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                                                                                                                                           | 26120/60000 [59:13:12<97:03:22, 10.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                                                                                                                                                           | 26140/60000 [59:16:51<102:12:37, 10.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                                                                                                                           | 26160/60000 [59:20:28<99:36:20, 10.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                                                                                                                          | 26179/60000 [59:24:01<100:50:54, 10.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                                                                                                                           | 26200/60000 [59:27:46<99:39:12, 10.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                                                                                                                          | 26219/60000 [59:31:13<103:37:30, 11.04s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                                                                                                                          | 26239/60000 [59:34:58<102:30:57, 10.93s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                                                                                                                                                                          | 26259/60000 [59:38:38<100:57:00, 10.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                                                                                                                          | 26279/60000 [59:42:33<133:37:48, 14.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                                                                                                                                                                          | 26299/60000 [59:46:26<99:23:57, 10.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                                                                                                                                                                          | 26319/60000 [59:50:01<96:12:37, 10.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                                                                                                                          | 26340/60000 [59:53:36<93:07:52,  9.96s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                                                                                                                                                                          | 26360/60000 [59:57:08<100:57:02, 10.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                                                                                                                                                                          | 26379/60000 [60:00:32<99:36:38, 10.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                                                                                                          | 26399/60000 [60:04:05<97:20:23, 10.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                                                                                                                         | 26420/60000 [60:07:53<102:31:13, 10.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                                                                                                                                                         | 26432/60000 [60:10:06<109:13:12, 11.71s/it]Traceback (most recent call last):
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/huggingface_hub/utils/_errors.py", line 270, in hf_raise_for_status
+    response.raise_for_status()
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+    raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://huggingface.co/datasets/Finnish-NLP/aalto_eduskunta_asr_audio_processed/resolve/3bdd45efc0b1a61f49cc79e94eb66e15c4432c89/data/train-00034-of-00064-2ff163d1bb53e2f0.parquet
+The above exception was the direct cause of the following exception:
+Traceback (most recent call last):
+  File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 679, in <module>
+    main()
+  File "/mnt/e/run_speech_recognition_seq2seq_streaming.py", line 628, in main
+    train_result = trainer.train(resume_from_checkpoint=checkpoint)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1547, in train
+    return inner_training_loop(
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/transformers/trainer.py", line 1839, in _inner_training_loop
+    for step, inputs in enumerate(epoch_iterator):
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/accelerate/data_loader.py", line 675, in __iter__
+    next_batch, next_batch_info = self._fetch_batches(main_iterator)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/accelerate/data_loader.py", line 604, in _fetch_batches
+    batches.append(next(iterator))
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 630, in __next__
+    data = self._next_data()
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 674, in _next_data
+    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 32, in fetch
+    data.append(next(self.dataset_iter))
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 1379, in __iter__
+    for key, example in ex_iterable:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 862, in __iter__
+    yield from self._iter()
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 899, in _iter
+    for key, example in iterator:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 982, in __iter__
+    for x in self.ex_iterable:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 678, in __iter__
+    yield from self._iter()
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 740, in _iter
+    for key, example in iterator:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 1114, in __iter__
+    for key, example in self.ex_iterable:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 429, in __iter__
+    if not iterators[i].hasnext():
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 106, in hasnext
+    self._thenext = next(self.it)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 678, in __iter__
+    yield from self._iter()
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 740, in _iter
+    for key, example in iterator:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 1114, in __iter__
+    for key, example in self.ex_iterable:
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/iterable_dataset.py", line 320, in __iter__
+    for key, pa_table in self.generate_tables_fn(**kwargs_with_shuffled_shards):
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/packaged_modules/parquet/parquet.py", line 85, in _generate_tables
+    parquet_file = pq.ParquetFile(f)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/pyarrow/parquet/core.py", line 341, in __init__
+    self.reader.open(
+  File "pyarrow/_parquet.pyx", line 1249, in pyarrow._parquet.ParquetReader.open
+  File "pyarrow/types.pxi", line 88, in pyarrow.lib._datatype_to_pep3118
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/datasets/download/streaming_download_manager.py", line 333, in read_with_retries
+    out = read(*args, **kwargs)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/fsspec/spec.py", line 1856, in read
+    out = self.cache._fetch(self.loc, self.loc + length)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/fsspec/caching.py", line 189, in _fetch
+    self.cache = self.fetcher(start, end)  # new block replaces old
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/huggingface_hub/hf_file_system.py", line 445, in _fetch_range
+    hf_raise_for_status(r)
+  File "/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/lib/python3.10/site-packages/huggingface_hub/utils/_errors.py", line 330, in hf_raise_for_status
+    raise HfHubHTTPError(str(e), response=response) from e
+huggingface_hub.utils._errors.HfHubHTTPError: 500 Server Error: Internal Server Error for url: https://huggingface.co/datasets/Finnish-NLP/aalto_eduskunta_asr_audio_processed/resolve/3bdd45efc0b1a61f49cc79e94eb66e15c4432c89/data/train-00034-of-00064-2ff163d1bb53e2f0.parquet (Request ID: Root=1-655d1c83-6ac0a7bb67f4b01056d447b6;974175f5-471f-4e9c-a910-35fa68eae047)
+Internal Error - We're working hard to fix this as soon as possible!
\ No newline at end of file