Training in progress, step 17500

errs

@@ -1973,3 +1973,49 @@ Configuration saved in ./checkpoint-17000/config.json
 Model weights saved in ./checkpoint-17000/pytorch_model.bin
 Configuration saved in ./checkpoint-17000/preprocessor_config.json
 Configuration saved in ./preprocessor_config.json
+Deleting older checkpoint [checkpoint-16500] due to args.save_total_limit
+
 94% 17001/18000 [46:11:46<19:28:06, 70.16s/it]
 94% 17002/18000 [46:11:59<14:40:27, 52.93s/it]
 94% 17003/18000 [46:12:10<11:09:38, 40.30s/it]
 94% 17004/18000 [46:12:19<8:34:04, 30.97s/it] 
 94% 17005/18000 [46:12:27<6:40:55, 24.18s/it]
 94% 17006/18000 [46:12:34<5:15:45, 19.06s/it]
 94% 17007/18000 [46:12:40<4:08:37, 15.02s/it]
 94% 17008/18000 [46:12:44<3:16:05, 11.86s/it]
 94% 17009/18000 [46:13:01<3:39:52, 13.31s/it]
 94% 17010/18000 [46:13:15<3:41:20, 13.42s/it]
 95% 17011/18000 [46:13:26<3:28:56, 12.68s/it]
 95% 17012/18000 [46:13:35<3:10:32, 11.57s/it]
 95% 17013/18000 [46:13:43<2:54:53, 10.63s/it]
 95% 17014/18000 [46:13:50<2:37:13,  9.57s/it]
 95% 17015/18000 [46:13:56<2:19:45,  8.51s/it]
 95% 17016/18000 [46:14:01<1:59:53,  7.31s/it]
 95% 17017/18000 [46:14:17<2:45:03, 10.08s/it]
 95% 17018/18000 [46:14:30<2:59:55, 10.99s/it]
 95% 17019/18000 [46:14:41<2:59:01, 10.95s/it]
 95% 17020/18000 [46:14:51<2:51:33, 10.50s/it]
 95% 17021/18000 [46:14:59<2:40:06,  9.81s/it]
 95% 17022/18000 [46:15:06<2:25:13,  8.91s/it]
 95% 17023/18000 [46:15:11<2:09:17,  7.94s/it]
 95% 17024/18000 [46:15:16<1:52:06,  6.89s/it]
 95% 17025/18000 [46:15:31<2:34:02,  9.48s/it]
 95% 17026/18000 [46:15:41<2:36:44,  9.66s/it]
 95% 17027/18000 [46:15:49<2:25:19,  8.96s/it]
 95% 17028/18000 [46:15:52<1:56:27,  7.19s/it]
 95% 17029/18000 [46:16:09<2:42:45, 10.06s/it]
 95% 17030/18000 [46:16:22<2:59:16, 11.09s/it]
 95% 17031/18000 [46:16:33<2:58:06, 11.03s/it]
 95% 17032/18000 [46:16:43<2:50:55, 10.59s/it]
 95% 17033/18000 [46:16:51<2:39:16,  9.88s/it]
 95% 17034/18000 [46:16:58<2:24:46,  8.99s/it]
 95% 17035/18000 [46:17:04<2:09:06,  8.03s/it]
 95% 17036/18000 [46:17:08<1:51:01,  6.91s/it]
 95% 17037/18000 [46:17:24<2:36:51,  9.77s/it]
 95% 17038/18000 [46:17:37<2:52:47, 10.78s/it]
 95% 17039/18000 [46:17:48<2:52:32, 10.77s/it]
 95% 17040/18000 [46:17:57<2:44:23, 10.27s/it]
 95% 17041/18000 [46:18:05<2:33:02,  9.57s/it]
 95% 17042/18000 [46:18:12<2:20:21,  8.79s/it]
 95% 17043/18000 [46:18:18<2:05:35,  7.87s/it]
 95% 17044/18000 [46:18:22<1:48:44,  6.83s/it]
 95% 17045/18000 [46:18:39<2:34:51,  9.73s/it]
 95% 17046/18000 [46:18:52<2:51:21, 10.78s/it]
 95% 17047/18000 [46:19:03<2:50:46, 10.75s/it]
 95% 17048/18000 [46:19:12<2:43:49, 10.32s/it]
 95% 17049/18000 [46:19:20<2:33:31,  9.69s/it]
 95% 17050/18000 [46:19:27<2:20:45,  8.89s/it]
 95% 17051/18000 [46:19:33<2:05:41,  7.95s/it]
 95% 17052/18000 [46:19:37<1:48:51,  6.89s/it]
 95% 17053/18000 [46:19:54<2:34:48,  9.81s/it]
 95% 17054/18000 [46:20:07<2:50:18, 10.80s/it]
 95% 17055/18000 [46:20:18<2:51:20, 10.88s/it]
 95% 17056/18000 [46:20:28<2:45:27, 10.52s/it]
 95% 17057/18000 [46:20:36<2:35:58,  9.92s/it]
 95% 17058/18000 [46:20:44<2:23:19,  9.13s/it]
 95% 17059/18000 [46:20:50<2:08:47,  8.21s/it]
 95% 17060/18000 [46:20:54<1:51:22,  7.11s/it]
 95% 17061/18000 [46:21:10<2:29:11,  9.53s/it]
 95% 17062/18000 [46:21:19<2:31:00,  9.66s/it]
 95% 17063/18000 [46:21:27<2:20:14,  8.98s/it]
 95% 17064/18000 [46:21:30<1:52:53,  7.24s/it]
 95% 17065/18000 [46:21:47<2:37:13, 10.09s/it]
 95% 17066/18000 [46:22:00<2:52:23, 11.07s/it]
 95% 17067/18000 [46:22:11<2:50:28, 10.96s/it]
 95% 17068/18000 [46:22:20<2:42:52, 10.49s/it]
 95% 17069/18000 [46:22:28<2:32:21,  9.82s/it]
 95% 17070/18000 [46:22:36<2:19:15,  8.98s/it]
 95% 17071/18000 [46:22:42<2:05:24,  8.10s/it]
 95% 17072/18000 [46:22:46<1:48:53,  7.04s/it]
 95% 17073/18000 [46:23:02<2:31:58,  9.84s/it]
 95% 17074/18000 [46:23:16<2:47:57, 10.88s/it]
 95% 17075/18000 [46:23:26<2:44:11, 10.65s/it]
 95% 17076/18000 [46:23:35<2:39:03, 10.33s/it]
 95% 17077/18000 [46:23:44<2:29:45,  9.73s/it]
 95% 17078/18000 [46:23:51<2:17:13,  8.93s/it]
 95% 17079/18000 [46:23:56<2:01:44,  7.93s/it]
 95% 17080/18000 [46:24:01<1:44:40,  6.83s/it]
 95% 17081/18000 [46:24:17<2:29:55,  9.79s/it]
 95% 17082/18000 [46:24:31<2:45:44, 10.83s/it]
 95% 17083/18000 [46:24:42<2:45:24, 10.82s/it]
 95% 17084/18000 [46:24:51<2:39:25, 10.44s/it]
 95% 17085/18000 [46:24:59<2:29:59,  9.84s/it]
 95% 17086/18000 [46:25:07<2:17:09,  9.00s/it]
 95% 17087/18000 [46:25:12<2:02:50,  8.07s/it]
 95% 17088/18000 [46:25:17<1:46:40,  7.02s/it]
 95% 17089/18000 [46:25:34<2:30:26,  9.91s/it]
 95% 17090/18000 [46:25:47<2:46:55, 11.01s/it]
 95% 17091/18000 [46:25:58<2:44:28, 10.86s/it]
 95% 17092/18000 [46:26:07<2:36:58, 10.37s/it]
 95% 17093/18000 [46:26:15<2:27:10,  9.74s/it]
 95% 17094/18000 [46:26:22<2:14:32,  8.91s/it]
 95% 17095/18000 [46:26:28<2:00:16,  7.97s/it]
 95% 17096/18000 [46:26:32<1:44:01,  6.90s/it]
 95% 17097/18000 [46:26:48<2:21:42,  9.42s/it]
 95% 17098/18000 [46:26:58<2:24:14,  9.59s/it]
 95% 17099/18000 [46:27:05<2:12:45,  8.84s/it]
 95% 17100/18000 [46:27:08<1:48:07,  7.21s/it]
                                              
 95% 17100/18000 [46:27:08<1:48:07,  7.21s/it]
 95% 17101/18000 [46:27:24<2:27:22,  9.84s/it]
 95% 17102/18000 [46:27:37<2:41:59, 10.82s/it]
 95% 17103/18000 [46:27:48<2:41:20, 10.79s/it]
 95% 17104/18000 [46:27:57<2:34:38, 10.36s/it]
 95% 17105/18000 [46:28:06<2:25:10,  9.73s/it]
 95% 17106/18000 [46:28:13<2:12:42,  8.91s/it]
 95% 17107/18000 [46:28:18<1:59:11,  8.01s/it]
 95% 17108/18000 [46:28:23<1:43:03,  6.93s/it]
 95% 17109/18000 [46:28:40<2:26:23,  9.86s/it]
 95% 17110/18000 [46:28:53<2:41:35, 10.89s/it]
 95% 17111/18000 [46:29:04<2:40:36, 10.84s/it]
 95% 17112/18000 [46:29:13<2:34:10, 10.42s/it]
 95% 17113/18000 [46:29:21<2:24:19,  9.76s/it]
 95% 17114/18000 [46:29:28<2:11:49,  8.93s/it]
 95% 17115/18000 [46:29:34<1:57:10,  7.94s/it]
 95% 17116/18000 [46:29:38<1:41:05,  6.86s/it]
 95% 17117/18000 [46:29:55<2:23:48,  9.77s/it]
 95% 17118/18000 [46:30:08<2:40:05, 10.89s/it]
 95% 17119/18000 [46:30:19<2:39:12, 10.84s/it]
 95% 17120/18000 [46:30:29<2:33:45, 10.48s/it]
 95% 17121/18000 [46:30:37<2:24:10,  9.84s/it]
 95% 17122/18000 [46:30:44<2:11:06,  8.96s/it]
 95% 17123/18000 [46:30:50<1:57:45,  8.06s/it]
 95% 17124/18000 [46:30:54<1:41:17,  6.94s/it]
 95% 17125/18000 [46:31:11<2:22:55,  9.80s/it]
 95% 17126/18000 [46:31:24<2:37:33, 10.82s/it]
 95% 17127/18000 [46:31:35<2:37:16, 10.81s/it]
 95% 17128/18000 [46:31:44<2:30:23, 10.35s/it]
 95% 17129/18000 [46:31:52<2:20:23,  9.67s/it]
 95% 17130/18000 [46:31:59<2:07:17,  8.78s/it]
 95% 17131/18000 [46:32:04<1:53:01,  7.80s/it]
 95% 17132/18000 [46:32:09<1:38:02,  6.78s/it]
 95% 17133/18000 [46:32:24<2:15:38,  9.39s/it]
 95% 17134/18000 [46:32:35<2:20:25,  9.73s/it]
 95% 17135/18000 [46:32:42<2:10:38,  9.06s/it]
 95% 17136/18000 [46:32:46<1:46:23,  7.39s/it]
 95% 17137/18000 [46:33:02<2:26:45, 10.20s/it]
 95% 17138/18000 [46:33:16<2:39:17, 11.09s/it]
 95% 17139/18000 [46:33:26<2:36:42, 10.92s/it]
 95% 17140/18000 [46:33:35<2:27:01, 10.26s/it]
 95% 17141/18000 [46:33:43<2:17:48,  9.63s/it]
 95% 17142/18000 [46:33:50<2:05:55,  8.81s/it]
 95% 17143/18000 [46:33:55<1:52:06,  7.85s/it]
 95% 17144/18000 [46:34:00<1:37:23,  6.83s/it]
 95% 17145/18000 [46:34:16<2:18:21,  9.71s/it]
 95% 17146/18000 [46:34:30<2:33:25, 10.78s/it]
 95% 17147/18000 [46:34:41<2:33:44, 10.81s/it]
 95% 17148/18000 [46:34:50<2:28:43, 10.47s/it]
 95% 17149/18000 [46:34:59<2:19:36,  9.84s/it]
 95% 17150/18000 [46:35:06<2:08:15,  9.05s/it]
 95% 17151/18000 [46:35:12<1:55:12,  8.14s/it]
 95% 17152/18000 [46:35:16<1:39:48,  7.06s/it]
 95% 17153/18000 [46:35:33<2:19:47,  9.90s/it]
 95% 17154/18000 [46:35:46<2:34:18, 10.94s/it]
 95% 17155/18000 [46:35:57<2:33:13, 10.88s/it]
 95% 17156/18000 [46:36:06<2:25:17, 10.33s/it]
 95% 17157/18000 [46:36:14<2:16:11,  9.69s/it]
 95% 17158/18000 [46:36:21<2:03:34,  8.81s/it]
 95% 17159/18000 [46:36:27<1:50:16,  7.87s/it]
 95% 17160/18000 [46:36:31<1:35:04,  6.79s/it]
 95% 17161/18000 [46:36:47<2:13:12,  9.53s/it]
 95% 17162/18000 [46:37:00<2:29:03, 10.67s/it]
 95% 17163/18000 [46:37:11<2:29:40, 10.73s/it]
 95% 17164/18000 [46:37:20<2:23:11, 10.28s/it]
 95% 17165/18000 [46:37:29<2:14:47,  9.69s/it]
 95% 17166/18000 [46:37:36<2:03:21,  8.87s/it]
 95% 17167/18000 [46:37:41<1:50:29,  7.96s/it]
 95% 17168/18000 [46:37:46<1:35:35,  6.89s/it]
 95% 17169/18000 [46:38:01<2:10:43,  9.44s/it]
 95% 17170/18000 [46:38:11<2:13:41,  9.66s/it]
 95% 17171/18000 [46:38:19<2:04:04,  8.98s/it]
 95% 17172/18000 [46:38:22<1:40:51,  7.31s/it]
 95% 17173/18000 [46:38:39<2:18:48, 10.07s/it]
 95% 17174/18000 [46:38:51<2:29:33, 10.86s/it]
 95% 17175/18000 [46:39:02<2:28:10, 10.78s/it]
 95% 17176/18000 [46:39:11<2:21:54, 10.33s/it]
 95% 17177/18000 [46:39:19<2:13:04,  9.70s/it]
 95% 17178/18000 [46:39:27<2:01:59,  8.91s/it]
 95% 17179/18000 [46:39:32<1:48:58,  7.96s/it]
 95% 17180/18000 [46:39:37<1:34:01,  6.88s/it]
 95% 17181/18000 [46:39:53<2:12:59,  9.74s/it]
 95% 17182/18000 [46:40:06<2:25:09, 10.65s/it]
 95% 17183/18000 [46:40:16<2:24:50, 10.64s/it]
 95% 17184/18000 [46:40:26<2:19:49, 10.28s/it]
 95% 17185/18000 [46:40:34<2:11:39,  9.69s/it]
 95% 17186/18000 [46:40:41<1:59:57,  8.84s/it]
 95% 17187/18000 [46:40:46<1:45:51,  7.81s/it]
 95% 17188/18000 [46:40:51<1:31:38,  6.77s/it]
 95% 17189/18000 [46:41:07<2:11:33,  9.73s/it]
 96% 17190/18000 [46:41:20<2:24:37, 10.71s/it]
 96% 17191/18000 [46:41:31<2:25:44, 10.81s/it]
 96% 17192/18000 [46:41:41<2:20:54, 10.46s/it]
 96% 17193/18000 [46:41:49<2:12:02,  9.82s/it]
 96% 17194/18000 [46:41:57<2:00:49,  8.99s/it]
 96% 17195/18000 [46:42:02<1:48:18,  8.07s/it]
 96% 17196/18000 [46:42:07<1:33:55,  7.01s/it]
 96% 17197/18000 [46:42:24<2:12:54,  9.93s/it]
 96% 17198/18000 [46:42:37<2:27:41, 11.05s/it]
 96% 17199/18000 [46:42:48<2:26:09, 10.95s/it]
 96% 17200/18000 [46:42:58<2:21:06, 10.58s/it]
                                              
 96% 17200/18000 [46:42:58<2:21:06, 10.58s/it]
 96% 17201/18000 [46:43:06<2:10:13,  9.78s/it]
 96% 17202/18000 [46:43:12<1:57:28,  8.83s/it]
 96% 17203/18000 [46:43:18<1:44:53,  7.90s/it]
 96% 17204/18000 [46:43:22<1:30:41,  6.84s/it]
 96% 17205/18000 [46:43:38<2:03:48,  9.34s/it]
 96% 17206/18000 [46:43:48<2:06:13,  9.54s/it]
 96% 17207/18000 [46:43:55<1:57:58,  8.93s/it]
 96% 17208/18000 [46:43:58<1:35:44,  7.25s/it]
 96% 17209/18000 [46:44:15<2:12:13, 10.03s/it]
 96% 17210/18000 [46:44:28<2:25:15, 11.03s/it]
 96% 17211/18000 [46:44:39<2:24:28, 10.99s/it]
 96% 17212/18000 [46:44:49<2:18:22, 10.54s/it]
 96% 17213/18000 [46:44:57<2:09:27,  9.87s/it]
 96% 17214/18000 [46:45:04<1:58:09,  9.02s/it]
 96% 17215/18000 [46:45:10<1:45:14,  8.04s/it]
 96% 17216/18000 [46:45:14<1:30:49,  6.95s/it]
 96% 17217/18000 [46:45:31<2:08:21,  9.84s/it]
 96% 17218/18000 [46:45:44<2:21:18, 10.84s/it]
 96% 17219/18000 [46:45:55<2:20:46, 10.82s/it]
 96% 17220/18000 [46:46:04<2:15:23, 10.41s/it]
 96% 17221/18000 [46:46:12<2:06:08,  9.72s/it]
 96% 17222/18000 [46:46:19<1:54:57,  8.87s/it]
 96% 17223/18000 [46:46:25<1:42:45,  7.94s/it]
 96% 17224/18000 [46:46:29<1:28:39,  6.86s/it]
 96% 17225/18000 [46:46:46<2:06:07,  9.77s/it]
 96% 17226/18000 [46:46:59<2:20:11, 10.87s/it]
 96% 17227/18000 [46:47:10<2:20:19, 10.89s/it]
 96% 17228/18000 [46:47:20<2:15:44, 10.55s/it]
 96% 17229/18000 [46:47:29<2:08:01,  9.96s/it]
 96% 17230/18000 [46:47:36<1:56:29,  9.08s/it]
 96% 17231/18000 [46:47:42<1:44:18,  8.14s/it]
 96% 17232/18000 [46:47:46<1:30:07,  7.04s/it]
 96% 17233/18000 [46:48:02<2:06:10,  9.87s/it]
 96% 17234/18000 [46:48:16<2:18:16, 10.83s/it]
 96% 17235/18000 [46:48:26<2:17:07, 10.75s/it]
 96% 17236/18000 [46:48:35<2:10:16, 10.23s/it]
 96% 17237/18000 [46:48:43<2:02:00,  9.59s/it]
 96% 17238/18000 [46:48:50<1:51:28,  8.78s/it]
 96% 17239/18000 [46:48:56<1:38:47,  7.79s/it]
 96% 17240/18000 [46:49:00<1:25:17,  6.73s/it]
 96% 17241/18000 [46:49:15<1:57:57,  9.32s/it]
 96% 17242/18000 [46:49:25<1:59:55,  9.49s/it]
 96% 17243/18000 [46:49:32<1:51:14,  8.82s/it]
 96% 17244/18000 [46:49:36<1:30:38,  7.19s/it]
 96% 17245/18000 [46:49:52<2:06:11, 10.03s/it]
 96% 17246/18000 [46:50:06<2:18:29, 11.02s/it]
 96% 17247/18000 [46:50:16<2:15:54, 10.83s/it]
 96% 17248/18000 [46:50:26<2:10:17, 10.40s/it]
 96% 17249/18000 [46:50:34<2:01:41,  9.72s/it]
 96% 17250/18000 [46:50:41<1:51:28,  8.92s/it]
 96% 17251/18000 [46:50:46<1:38:22,  7.88s/it]
 96% 17252/18000 [46:50:51<1:25:02,  6.82s/it]
 96% 17253/18000 [46:51:07<2:01:23,  9.75s/it]
 96% 17254/18000 [46:51:21<2:14:59, 10.86s/it]
 96% 17255/18000 [46:51:31<2:13:35, 10.76s/it]
 96% 17256/18000 [46:51:40<2:07:51, 10.31s/it]
 96% 17257/18000 [46:51:49<2:00:51,  9.76s/it]
 96% 17258/18000 [46:51:56<1:49:58,  8.89s/it]
 96% 17259/18000 [46:52:01<1:38:13,  7.95s/it]
 96% 17260/18000 [46:52:06<1:24:51,  6.88s/it]
 96% 17261/18000 [46:52:22<1:59:28,  9.70s/it]
 96% 17262/18000 [46:52:35<2:11:58, 10.73s/it]
 96% 17263/18000 [46:52:46<2:11:39, 10.72s/it]
 96% 17264/18000 [46:52:55<2:06:42, 10.33s/it]
 96% 17265/18000 [46:53:03<1:58:30,  9.67s/it]
 96% 17266/18000 [46:53:10<1:47:37,  8.80s/it]
 96% 17267/18000 [46:53:16<1:36:40,  7.91s/it]
 96% 17268/18000 [46:53:21<1:24:10,  6.90s/it]
 96% 17269/18000 [46:53:37<1:59:22,  9.80s/it]
 96% 17270/18000 [46:53:51<2:12:13, 10.87s/it]
 96% 17271/18000 [46:54:01<2:11:59, 10.86s/it]
 96% 17272/18000 [46:54:11<2:06:27, 10.42s/it]
 96% 17273/18000 [46:54:19<1:58:08,  9.75s/it]
 96% 17274/18000 [46:54:26<1:48:01,  8.93s/it]
 96% 17275/18000 [46:54:32<1:35:40,  7.92s/it]
 96% 17276/18000 [46:54:36<1:22:27,  6.83s/it]
 96% 17277/18000 [46:54:51<1:52:51,  9.37s/it]
 96% 17278/18000 [46:55:01<1:54:59,  9.56s/it]
 96% 17279/18000 [46:55:09<1:47:13,  8.92s/it]
 96% 17280/18000 [46:55:12<1:26:20,  7.19s/it]
 96% 17281/18000 [46:55:28<2:00:08, 10.03s/it]
 96% 17282/18000 [46:55:42<2:12:19, 11.06s/it]
 96% 17283/18000 [46:55:53<2:11:45, 11.03s/it]
 96% 17284/18000 [46:56:02<2:04:47, 10.46s/it]
 96% 17285/18000 [46:56:10<1:56:24,  9.77s/it]
 96% 17286/18000 [46:56:17<1:46:04,  8.91s/it]
 96% 17287/18000 [46:56:22<1:33:30,  7.87s/it]
 96% 17288/18000 [46:56:27<1:21:04,  6.83s/it]
 96% 17289/18000 [46:56:43<1:55:11,  9.72s/it]
 96% 17290/18000 [46:56:57<2:07:57, 10.81s/it]
 96% 17291/18000 [46:57:08<2:08:04, 10.84s/it]
 96% 17292/18000 [46:57:17<2:03:13, 10.44s/it]
 96% 17293/18000 [46:57:25<1:54:56,  9.75s/it]
 96% 17294/18000 [46:57:32<1:44:40,  8.90s/it]
 96% 17295/18000 [46:57:38<1:33:00,  7.92s/it]
 96% 17296/18000 [46:57:42<1:19:31,  6.78s/it]
 96% 17297/18000 [46:57:58<1:53:41,  9.70s/it]
 96% 17298/18000 [46:58:12<2:05:53, 10.76s/it]
 96% 17299/18000 [46:58:22<2:04:54, 10.69s/it]
 96% 17300/18000 [46:58:32<2:00:21, 10.32s/it]
                                              
 96% 17300/18000 [46:58:32<2:00:21, 10.32s/it]
 96% 17301/18000 [46:58:40<1:52:45,  9.68s/it]
 96% 17302/18000 [46:58:47<1:42:46,  8.84s/it]
 96% 17303/18000 [46:58:52<1:31:57,  7.92s/it]
 96% 17304/18000 [46:58:57<1:19:30,  6.85s/it]
 96% 17305/18000 [46:59:13<1:52:28,  9.71s/it]
 96% 17306/18000 [46:59:27<2:05:12, 10.82s/it]
 96% 17307/18000 [46:59:37<2:03:53, 10.73s/it]
 96% 17308/18000 [46:59:47<1:59:24, 10.35s/it]
 96% 17309/18000 [46:59:55<1:52:19,  9.75s/it]
 96% 17310/18000 [47:00:02<1:42:50,  8.94s/it]
 96% 17311/18000 [47:00:08<1:32:10,  8.03s/it]
 96% 17312/18000 [47:00:12<1:19:38,  6.95s/it]
 96% 17313/18000 [47:00:27<1:47:01,  9.35s/it]
 96% 17314/18000 [47:00:37<1:48:29,  9.49s/it]
 96% 17315/18000 [47:00:44<1:40:37,  8.81s/it]
 96% 17316/18000 [47:00:48<1:21:55,  7.19s/it]
 96% 17317/18000 [47:01:04<1:53:31,  9.97s/it]
 96% 17318/18000 [47:01:17<2:04:48, 10.98s/it]
 96% 17319/18000 [47:01:28<2:04:07, 10.94s/it]
 96% 17320/18000 [47:01:38<1:59:15, 10.52s/it]
 96% 17321/18000 [47:01:46<1:51:36,  9.86s/it]
 96% 17322/18000 [47:01:53<1:41:09,  8.95s/it]
 96% 17323/18000 [47:01:59<1:30:10,  7.99s/it]
 96% 17324/18000 [47:02:03<1:18:11,  6.94s/it]
 96% 17325/18000 [47:02:20<1:49:25,  9.73s/it]
 96% 17326/18000 [47:02:33<2:00:40, 10.74s/it]
 96% 17327/18000 [47:02:43<2:00:28, 10.74s/it]
 96% 17328/18000 [47:02:53<1:55:28, 10.31s/it]
 96% 17329/18000 [47:03:01<1:47:59,  9.66s/it]
 96% 17330/18000 [47:03:08<1:38:12,  8.79s/it]
 96% 17331/18000 [47:03:13<1:26:39,  7.77s/it]
 96% 17332/18000 [47:03:17<1:14:51,  6.72s/it]
 96% 17333/18000 [47:03:34<1:47:16,  9.65s/it]
 96% 17334/18000 [47:03:47<1:58:35, 10.68s/it]
 96% 17335/18000 [47:03:58<1:58:36, 10.70s/it]
 96% 17336/18000 [47:04:07<1:54:26, 10.34s/it]
 96% 17337/18000 [47:04:15<1:47:32,  9.73s/it]
 96% 17338/18000 [47:04:22<1:38:37,  8.94s/it]
 96% 17339/18000 [47:04:28<1:28:12,  8.01s/it]
 96% 17340/18000 [47:04:33<1:16:17,  6.94s/it]
 96% 17341/18000 [47:04:49<1:47:50,  9.82s/it]
 96% 17342/18000 [47:05:03<1:59:05, 10.86s/it]
 96% 17343/18000 [47:05:13<1:58:18, 10.80s/it]
 96% 17344/18000 [47:05:23<1:53:08, 10.35s/it]
 96% 17345/18000 [47:05:31<1:45:34,  9.67s/it]
 96% 17346/18000 [47:05:37<1:35:30,  8.76s/it]
 96% 17347/18000 [47:05:43<1:25:14,  7.83s/it]
 96% 17348/18000 [47:05:47<1:13:33,  6.77s/it]
 96% 17349/18000 [47:06:02<1:40:46,  9.29s/it]
 96% 17350/18000 [47:06:12<1:42:02,  9.42s/it]
 96% 17351/18000 [47:06:19<1:34:40,  8.75s/it]
 96% 17352/18000 [47:06:23<1:16:49,  7.11s/it]
 96% 17353/18000 [47:06:39<1:47:03,  9.93s/it]
 96% 17354/18000 [47:06:52<1:57:44, 10.94s/it]
 96% 17355/18000 [47:07:03<1:56:53, 10.87s/it]
 96% 17356/18000 [47:07:12<1:51:50, 10.42s/it]
 96% 17357/18000 [47:07:21<1:44:37,  9.76s/it]
 96% 17358/18000 [47:07:27<1:34:56,  8.87s/it]
 96% 17359/18000 [47:07:33<1:24:02,  7.87s/it]
 96% 17360/18000 [47:07:37<1:12:19,  6.78s/it]
 96% 17361/18000 [47:07:54<1:42:40,  9.64s/it]
 96% 17362/18000 [47:08:07<1:53:06, 10.64s/it]
 96% 17363/18000 [47:08:17<1:52:56, 10.64s/it]
 96% 17364/18000 [47:08:26<1:48:07, 10.20s/it]
 96% 17365/18000 [47:08:34<1:40:42,  9.52s/it]
 96% 17366/18000 [47:08:41<1:32:12,  8.73s/it]
 96% 17367/18000 [47:08:47<1:22:35,  7.83s/it]
 96% 17368/18000 [47:08:51<1:11:18,  6.77s/it]
 96% 17369/18000 [47:09:08<1:41:37,  9.66s/it]
 96% 17370/18000 [47:09:21<1:52:58, 10.76s/it]
 97% 17371/18000 [47:09:32<1:53:16, 10.80s/it]
 97% 17372/18000 [47:09:41<1:49:09, 10.43s/it]
 97% 17373/18000 [47:09:50<1:42:25,  9.80s/it]
 97% 17374/18000 [47:09:57<1:33:56,  9.00s/it]
 97% 17375/18000 [47:10:03<1:23:57,  8.06s/it]
 97% 17376/18000 [47:10:07<1:12:50,  7.00s/it]
 97% 17377/18000 [47:10:24<1:42:14,  9.85s/it]
 97% 17378/18000 [47:10:37<1:52:53, 10.89s/it]
 97% 17379/18000 [47:10:48<1:52:15, 10.85s/it]
 97% 17380/18000 [47:10:57<1:47:10, 10.37s/it]
 97% 17381/18000 [47:11:05<1:40:02,  9.70s/it]
 97% 17382/18000 [47:11:12<1:31:07,  8.85s/it]
 97% 17383/18000 [47:11:18<1:21:23,  7.91s/it]
 97% 17384/18000 [47:11:22<1:10:27,  6.86s/it]
 97% 17385/18000 [47:11:38<1:36:42,  9.43s/it]
 97% 17386/18000 [47:11:48<1:38:18,  9.61s/it]
 97% 17387/18000 [47:11:55<1:31:14,  8.93s/it]
 97% 17388/18000 [47:11:58<1:13:41,  7.22s/it]
 97% 17389/18000 [47:12:15<1:41:32,  9.97s/it]
 97% 17390/18000 [47:12:28<1:50:39, 10.88s/it]
 97% 17391/18000 [47:12:38<1:49:43, 10.81s/it]
 97% 17392/18000 [47:12:48<1:45:09, 10.38s/it]
 97% 17393/18000 [47:12:56<1:38:28,  9.73s/it]
 97% 17394/18000 [47:13:03<1:29:40,  8.88s/it]
 97% 17395/18000 [47:13:08<1:19:54,  7.92s/it]
 97% 17396/18000 [47:13:13<1:08:49,  6.84s/it]
 97% 17397/18000 [47:13:29<1:38:03,  9.76s/it]
 97% 17398/18000 [47:13:43<1:49:14, 10.89s/it]
 97% 17399/18000 [47:13:54<1:49:13, 10.90s/it]
 97% 17400/18000 [47:14:04<1:45:30, 10.55s/it]
                                              
 97% 17400/18000 [47:14:04<1:45:30, 10.55s/it]
 97% 17401/18000 [47:14:12<1:38:39,  9.88s/it]
 97% 17402/18000 [47:14:19<1:29:25,  8.97s/it]
 97% 17403/18000 [47:14:25<1:20:35,  8.10s/it]
 97% 17404/18000 [47:14:29<1:09:52,  7.03s/it]
 97% 17405/18000 [47:14:46<1:37:50,  9.87s/it]
 97% 17406/18000 [47:14:59<1:47:44, 10.88s/it]
 97% 17407/18000 [47:15:10<1:46:50, 10.81s/it]
 97% 17408/18000 [47:15:19<1:42:11, 10.36s/it]
 97% 17409/18000 [47:15:27<1:35:27,  9.69s/it]
 97% 17410/18000 [47:15:34<1:26:52,  8.84s/it]
 97% 17411/18000 [47:15:39<1:17:00,  7.84s/it]
 97% 17412/18000 [47:15:44<1:06:27,  6.78s/it]
 97% 17413/18000 [47:16:00<1:34:53,  9.70s/it]
 97% 17414/18000 [47:16:14<1:45:41, 10.82s/it]
 97% 17415/18000 [47:16:25<1:45:39, 10.84s/it]
 97% 17416/18000 [47:16:34<1:41:31, 10.43s/it]
 97% 17417/18000 [47:16:42<1:35:07,  9.79s/it]
 97% 17418/18000 [47:16:49<1:26:50,  8.95s/it]
 97% 17419/18000 [47:16:55<1:17:50,  8.04s/it]
 97% 17420/18000 [47:17:00<1:07:12,  6.95s/it]
 97% 17421/18000 [47:17:15<1:30:44,  9.40s/it]
 97% 17422/18000 [47:17:25<1:31:36,  9.51s/it]
 97% 17423/18000 [47:17:32<1:24:43,  8.81s/it]
 97% 17424/18000 [47:17:35<1:07:56,  7.08s/it]
 97% 17425/18000 [47:17:52<1:35:41,  9.99s/it]
 97% 17426/18000 [47:18:05<1:46:08, 11.09s/it]
 97% 17427/18000 [47:18:16<1:45:23, 11.04s/it]
 97% 17428/18000 [47:18:25<1:39:39, 10.45s/it]
 97% 17429/18000 [47:18:33<1:32:51,  9.76s/it]
 97% 17430/18000 [47:18:40<1:24:32,  8.90s/it]
 97% 17431/18000 [47:18:46<1:15:30,  7.96s/it]
 97% 17432/18000 [47:18:50<1:05:05,  6.88s/it]
 97% 17433/18000 [47:19:07<1:31:31,  9.68s/it]
 97% 17434/18000 [47:19:19<1:39:17, 10.53s/it]
 97% 17435/18000 [47:19:30<1:39:28, 10.56s/it]
 97% 17436/18000 [47:19:39<1:35:17, 10.14s/it]
 97% 17437/18000 [47:19:47<1:29:58,  9.59s/it]
 97% 17438/18000 [47:19:54<1:22:37,  8.82s/it]
 97% 17439/18000 [47:20:00<1:13:44,  7.89s/it]
 97% 17440/18000 [47:20:04<1:03:46,  6.83s/it]
 97% 17441/18000 [47:20:21<1:30:45,  9.74s/it]
 97% 17442/18000 [47:20:34<1:41:13, 10.88s/it]
 97% 17443/18000 [47:20:45<1:41:33, 10.94s/it]
 97% 17444/18000 [47:20:55<1:37:50, 10.56s/it]
 97% 17445/18000 [47:21:04<1:31:45,  9.92s/it]
 97% 17446/18000 [47:21:11<1:23:40,  9.06s/it]
 97% 17447/18000 [47:21:16<1:14:27,  8.08s/it]
 97% 17448/18000 [47:21:21<1:04:09,  6.97s/it]
 97% 17449/18000 [47:21:37<1:29:45,  9.77s/it]
 97% 17450/18000 [47:21:50<1:39:26, 10.85s/it]
 97% 17451/18000 [47:22:01<1:39:05, 10.83s/it]
 97% 17452/18000 [47:22:11<1:34:37, 10.36s/it]
 97% 17453/18000 [47:22:18<1:27:13,  9.57s/it]
 97% 17454/18000 [47:22:25<1:20:09,  8.81s/it]
 97% 17455/18000 [47:22:31<1:11:51,  7.91s/it]
 97% 17456/18000 [47:22:36<1:02:31,  6.90s/it]
 97% 17457/18000 [47:22:51<1:24:43,  9.36s/it]
 97% 17458/18000 [47:23:00<1:25:07,  9.42s/it]
 97% 17459/18000 [47:23:07<1:17:25,  8.59s/it]
 97% 17460/18000 [47:23:10<1:02:47,  6.98s/it]
 97% 17461/18000 [47:23:27<1:28:31,  9.85s/it]
 97% 17462/18000 [47:23:40<1:37:56, 10.92s/it]
 97% 17463/18000 [47:23:51<1:37:42, 10.92s/it]
 97% 17464/18000 [47:24:00<1:32:46, 10.39s/it]
 97% 17465/18000 [47:24:09<1:27:11,  9.78s/it]
 97% 17466/18000 [47:24:16<1:19:42,  8.96s/it]
 97% 17467/18000 [47:24:21<1:11:18,  8.03s/it]
 97% 17468/18000 [47:24:26<1:01:47,  6.97s/it]
 97% 17469/18000 [47:24:43<1:27:09,  9.85s/it]
 97% 17470/18000 [47:24:56<1:36:27, 10.92s/it]
 97% 17471/18000 [47:25:07<1:36:00, 10.89s/it]
 97% 17472/18000 [47:25:16<1:31:43, 10.42s/it]
 97% 17473/18000 [47:25:24<1:26:02,  9.80s/it]
 97% 17474/18000 [47:25:31<1:18:34,  8.96s/it]
 97% 17475/18000 [47:25:37<1:10:10,  8.02s/it]
 97% 17476/18000 [47:25:42<1:00:44,  6.96s/it]
 97% 17477/18000 [47:25:58<1:26:05,  9.88s/it]
 97% 17478/18000 [47:26:12<1:35:22, 10.96s/it]
 97% 17479/18000 [47:26:23<1:34:41, 10.91s/it]
 97% 17480/18000 [47:26:32<1:30:25, 10.43s/it]
 97% 17481/18000 [47:26:40<1:24:28,  9.77s/it]
 97% 17482/18000 [47:26:47<1:16:31,  8.86s/it]
 97% 17483/18000 [47:26:53<1:08:07,  7.91s/it]
 97% 17484/18000 [47:26:57<58:40,  6.82s/it]  
 97% 17485/18000 [47:27:13<1:23:25,  9.72s/it]
 97% 17486/18000 [47:27:27<1:32:10, 10.76s/it]
 97% 17487/18000 [47:27:37<1:32:02, 10.76s/it]
 97% 17488/18000 [47:27:47<1:28:21, 10.35s/it]
 97% 17489/18000 [47:27:55<1:22:48,  9.72s/it]
 97% 17490/18000 [47:28:02<1:15:27,  8.88s/it]
 97% 17491/18000 [47:28:08<1:07:12,  7.92s/it]
 97% 17492/18000 [47:28:12<57:50,  6.83s/it]  
 97% 17493/18000 [47:28:27<1:18:22,  9.28s/it]
 97% 17494/18000 [47:28:36<1:18:31,  9.31s/it]
 97% 17495/18000 [47:28:43<1:12:35,  8.63s/it]
 97% 17496/18000 [47:28:47<59:05,  7.03s/it]  
 97% 17497/18000 [47:29:03<1:22:52,  9.89s/it]
 97% 17498/18000 [47:29:17<1:31:41, 10.96s/it]
 97% 17499/18000 [47:29:27<1:30:18, 10.82s/it]
 97% 17500/18000 [47:29:36<1:26:14, 10.35s/it]
                                              
 97% 17500/18000 [47:29:36<1:26:14, 10.35s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+
+
  0% 0/33 [00:00<?, ?it/s]
+
  6% 2/33 [00:04<01:06,  2.16s/it]
+
  9% 3/33 [00:08<01:31,  3.05s/it]
+
 12% 4/33 [00:12<01:38,  3.38s/it]
+
 15% 5/33 [00:16<01:40,  3.59s/it]
+
 18% 6/33 [00:20<01:41,  3.75s/it]
+
 21% 7/33 [00:24<01:37,  3.74s/it]
+
 24% 8/33 [00:28<01:34,  3.79s/it]
+
 27% 9/33 [00:32<01:31,  3.83s/it]
+
 30% 10/33 [00:36<01:32,  4.03s/it]
+
 33% 11/33 [00:40<01:28,  4.01s/it]
+
 36% 12/33 [00:44<01:23,  3.98s/it]
+
 39% 13/33 [00:48<01:18,  3.92s/it]
+
 42% 14/33 [00:52<01:15,  3.97s/it]
+
 45% 15/33 [00:56<01:11,  3.98s/it]
+
 48% 16/33 [01:00<01:09,  4.11s/it]
+
 52% 17/33 [01:05<01:07,  4.24s/it]
+
 55% 18/33 [01:09<01:02,  4.18s/it]
+
 58% 19/33 [01:13<00:58,  4.18s/it]
+
 61% 20/33 [01:17<00:55,  4.25s/it]
+
 64% 21/33 [01:22<00:51,  4.30s/it]
+
 67% 22/33 [01:26<00:47,  4.29s/it]
+
 70% 23/33 [01:31<00:43,  4.33s/it]
+
 73% 24/33 [01:35<00:39,  4.34s/it]
+
 76% 25/33 [01:39<00:35,  4.38s/it]
+
 79% 26/33 [01:44<00:30,  4.33s/it]
+
 82% 27/33 [01:48<00:25,  4.29s/it]
+
 85% 28/33 [01:52<00:21,  4.31s/it]
+
 88% 29/33 [01:56<00:16,  4.06s/it]
+
 91% 30/33 [01:58<00:11,  3.67s/it]
+
 94% 31/33 [02:01<00:06,  3.39s/it]
+
 97% 32/33 [02:04<00:03,  3.15s/it]
+
                                              
+
                                   
 97% 17500/18000 [47:31:48<1:26:14, 10.35s/it]
+
+
                                   Saving model checkpoint to ./checkpoint-17500
+Configuration saved in ./checkpoint-17500/config.json
+Model weights saved in ./checkpoint-17500/pytorch_model.bin
+Configuration saved in ./checkpoint-17500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json

logs

@@ -296,3 +296,9 @@ xpu_backend=None,
 {'loss': 0.0117, 'learning_rate': 1.1614654919452406e-05, 'epoch': 466.67}
 {'loss': 0.0112, 'learning_rate': 9.770015053337078e-06, 'epoch': 469.44}
 {'loss': 0.0115, 'learning_rate': 8.083358732646206e-06, 'epoch': 472.22}
+{'eval_loss': 0.6556624174118042, 'eval_wer': 0.4069360483614381, 'eval_runtime': 132.4623, 'eval_samples_per_second': 19.696, 'eval_steps_per_second': 0.249, 'epoch': 472.22}
+{'loss': 0.0111, 'learning_rate': 6.555229506231608e-06, 'epoch': 475.0}
+{'loss': 0.0118, 'learning_rate': 5.186119835347169e-06, 'epoch': 477.78}
+{'loss': 0.0112, 'learning_rate': 3.976470934946075e-06, 'epoch': 480.56}
+{'loss': 0.0117, 'learning_rate': 2.92667263149321e-06, 'epoch': 483.33}
+{'loss': 0.0111, 'learning_rate': 2.037063237338044e-06, 'epoch': 486.11}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99c126ed3f17fbfa6bba304de7aaff6c6c6e507e6c3c51039c6b256a52859ad2
+oid sha256:0df405d0e1ea0aa1d9a1b4a093c3d7582202187d5bac0e54b3b74e263b89570b
 size 1262083569

wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b2a97f3d80a727dd473279650caac8b63460053bec7dbe0456f71484a9ffbe9
-size 117590871
+oid sha256:8e2f2f8c8a99a57b77cfe4a9a380d00f976d1c0176f17bde23883e81badc72b7
+size 121132803