diff --git "a/nohup.out" "b/nohup.out" --- "a/nohup.out" +++ "b/nohup.out" @@ -1932,3 +1932,3768 @@ Configuration saved in ./checkpoint-9500/config.json Model weights saved in ./checkpoint-9500/pytorch_model.bin Configuration saved in ./checkpoint-9500/preprocessor_config.json Configuration saved in ./preprocessor_config.json +Deleting older checkpoint [checkpoint-8000] due to args.save_total_limit + 27%|██▋ | 9501/34750 [8:58:35<272:15:30, 38.82s/it] 27%|██▋ | 9502/34750 [8:58:39<198:09:17, 28.25s/it] 27%|██▋ | 9503/34750 [8:58:42<146:10:46, 20.84s/it] 27%|██▋ | 9504/34750 [8:58:46<109:54:11, 15.67s/it] 27%|██▋ | 9505/34750 [8:58:49<84:15:57, 12.02s/it] 27%|██▋ | 9506/34750 [8:58:53<66:02:48, 9.42s/it] 27%|██▋ | 9507/34750 [8:58:56<52:41:25, 7.51s/it] 27%|██▋ | 9508/34750 [8:58:59<43:38:22, 6.22s/it] 27%|██▋ | 9509/34750 [8:59:02<37:06:03, 5.29s/it] 27%|██▋ | 9510/34750 [8:59:05<32:26:34, 4.63s/it] 27%|██▋ | 9511/34750 [8:59:08<28:52:29, 4.12s/it] 27%|██▋ | 9512/34750 [8:59:11<26:15:41, 3.75s/it] 27%|██▋ | 9513/34750 [8:59:14<24:25:53, 3.49s/it] 27%|██▋ | 9514/34750 [8:59:17<23:17:09, 3.32s/it] 27%|██▋ | 9515/34750 [8:59:20<22:08:49, 3.16s/it] 27%|██▋ | 9516/34750 [8:59:22<21:21:19, 3.05s/it] 27%|██▋ | 9517/34750 [8:59:25<21:00:39, 3.00s/it] 27%|██▋ | 9518/34750 [8:59:28<20:12:11, 2.88s/it] 27%|██▋ | 9519/34750 [8:59:30<19:26:33, 2.77s/it] 27%|██▋ | 9520/34750 [8:59:33<18:51:05, 2.69s/it] 27%|██▋ | 9521/34750 [8:59:35<18:25:51, 2.63s/it] 27%|██▋ | 9522/34750 [8:59:38<18:04:19, 2.58s/it] 27%|██▋ | 9523/34750 [8:59:40<17:36:07, 2.51s/it] 27%|██▋ | 9524/34750 [8:59:42<17:02:47, 2.43s/it] 27%|██▋ | 9525/34750 [8:59:45<16:31:15, 2.36s/it] 27%|██▋ | 9526/34750 [8:59:47<16:11:10, 2.31s/it] 27%|██▋ | 9527/34750 [8:59:49<15:50:52, 2.26s/it] 27%|██▋ | 9528/34750 [8:59:51<15:45:54, 2.25s/it] 27%|██▋ | 9529/34750 [8:59:53<15:30:53, 2.21s/it] 27%|██▋ | 9530/34750 [8:59:55<15:09:20, 2.16s/it] 27%|██▋ | 9531/34750 [8:59:57<15:05:08, 2.15s/it] 27%|██▋ | 9532/34750 [8:59:59<14:39:26, 2.09s/it] 27%|██▋ | 9533/34750 [9:00:01<14:34:08, 2.08s/it] 27%|██▋ | 9534/34750 [9:00:03<13:57:58, 1.99s/it] 27%|██▋ | 9535/34750 [9:00:05<13:18:07, 1.90s/it] 27%|██▋ | 9536/34750 [9:00:12<23:50:22, 3.40s/it] 27%|██▋ | 9537/34750 [9:00:18<29:35:27, 4.23s/it] 27%|██▋ | 9538/34750 [9:00:23<31:40:18, 4.52s/it] 27%|██▋ | 9539/34750 [9:00:29<33:39:37, 4.81s/it] 27%|██▋ | 9540/34750 [9:00:34<34:18:20, 4.90s/it] 27%|██▋ | 9541/34750 [9:00:39<34:04:05, 4.87s/it] 27%|██▋ | 9542/34750 [9:00:43<33:13:52, 4.75s/it] 27%|██▋ | 9543/34750 [9:00:47<32:31:46, 4.65s/it] 27%|██▋ | 9544/34750 [9:00:52<31:56:30, 4.56s/it] 27%|██▋ | 9545/34750 [9:00:56<31:23:47, 4.48s/it] 27%|██▋ | 9546/34750 [9:01:00<30:40:54, 4.38s/it] 27%|██▋ | 9547/34750 [9:01:04<29:46:06, 4.25s/it] 27%|██▋ | 9548/34750 [9:01:08<28:46:09, 4.11s/it] 27%|██▋ | 9549/34750 [9:01:12<27:46:31, 3.97s/it] 27%|██▋ | 9550/34750 [9:01:15<26:45:29, 3.82s/it] 27%|██▋ | 9551/34750 [9:01:19<26:04:51, 3.73s/it] 27%|██▋ | 9552/34750 [9:01:22<25:25:58, 3.63s/it] 27%|██▋ | 9553/34750 [9:01:26<25:42:51, 3.67s/it] 27%|██▋ | 9554/34750 [9:01:29<25:01:31, 3.58s/it] 27%|██▋ | 9555/34750 [9:01:33<25:08:53, 3.59s/it] 27%|██▋ | 9556/34750 [9:01:36<24:26:12, 3.49s/it] 28%|██▊ | 9557/34750 [9:01:39<23:36:15, 3.37s/it] 28%|██▊ | 9558/34750 [9:01:42<23:03:36, 3.30s/it] 28%|██▊ | 9559/34750 [9:01:45<22:28:36, 3.21s/it] 28%|██▊ | 9560/34750 [9:01:48<22:00:08, 3.14s/it] 28%|██▊ | 9561/34750 [9:01:51<21:58:51, 3.14s/it] 28%|██▊ | 9562/34750 [9:01:54<21:39:44, 3.10s/it] 28%|██▊ | 9563/34750 [9:01:57<21:17:58, 3.04s/it] 28%|██▊ | 9564/34750 [9:02:00<21:03:07, 3.01s/it] 28%|██▊ | 9565/34750 [9:02:03<20:24:35, 2.92s/it] 28%|██▊ | 9566/34750 [9:02:06<19:43:24, 2.82s/it] 28%|██▊ | 9567/34750 [9:02:08<19:24:10, 2.77s/it] 28%|██▊ | 9568/34750 [9:02:11<19:07:57, 2.74s/it] 28%|██▊ | 9569/34750 [9:02:14<18:57:10, 2.71s/it] 28%|██▊ | 9570/34750 [9:02:16<18:54:30, 2.70s/it] 28%|██▊ | 9571/34750 [9:02:19<18:21:07, 2.62s/it] 28%|██▊ | 9572/34750 [9:02:21<18:03:29, 2.58s/it] 28%|██▊ | 9573/34750 [9:02:23<17:28:43, 2.50s/it] 28%|██▊ | 9574/34750 [9:02:26<17:09:40, 2.45s/it] 28%|██▊ | 9575/34750 [9:02:28<16:34:10, 2.37s/it] 28%|██▊ | 9576/34750 [9:02:30<16:35:17, 2.37s/it] 28%|██▊ | 9577/34750 [9:02:33<16:13:08, 2.32s/it] 28%|██▊ | 9578/34750 [9:02:35<15:47:36, 2.26s/it] 28%|██▊ | 9579/34750 [9:02:37<15:19:54, 2.19s/it] 28%|██▊ | 9580/34750 [9:02:39<15:01:49, 2.15s/it] 28%|██▊ | 9581/34750 [9:02:41<14:36:51, 2.09s/it] 28%|██▊ | 9582/34750 [9:02:43<14:11:38, 2.03s/it] 28%|██▊ | 9583/34750 [9:02:44<13:44:30, 1.97s/it] 28%|██▊ | 9584/34750 [9:02:46<13:11:32, 1.89s/it] 28%|██▊ | 9585/34750 [9:02:48<12:48:54, 1.83s/it] 28%|██▊ | 9586/34750 [9:02:54<21:44:13, 3.11s/it] 28%|██▊ | 9587/34750 [9:03:00<27:14:05, 3.90s/it] 28%|██▊ | 9588/34750 [9:03:05<29:48:06, 4.26s/it] 28%|██▊ | 9589/34750 [9:03:09<30:43:26, 4.40s/it] 28%|██▊ | 9590/34750 [9:03:14<30:58:01, 4.43s/it] 28%|██▊ | 9591/34750 [9:03:18<30:42:52, 4.39s/it] 28%|██▊ | 9592/34750 [9:03:23<30:22:00, 4.35s/it] 28%|██▊ | 9593/34750 [9:03:27<30:05:35, 4.31s/it] 28%|██▊ | 9594/34750 [9:03:31<29:28:24, 4.22s/it] 28%|██▊ | 9595/34750 [9:03:35<28:37:09, 4.10s/it] 28%|██▊ | 9596/34750 [9:03:39<28:24:30, 4.07s/it] 28%|██▊ | 9597/34750 [9:03:42<27:42:44, 3.97s/it] 28%|██▊ | 9598/34750 [9:03:46<27:10:17, 3.89s/it] 28%|██▊ | 9599/34750 [9:03:50<26:26:08, 3.78s/it] 28%|██▊ | 9600/34750 [9:03:53<26:22:23, 3.78s/it] 28%|██▊ | 9600/34750 [9:03:53<26:22:23, 3.78s/it] 28%|██▊ | 9601/34750 [9:03:57<25:49:03, 3.70s/it] 28%|██▊ | 9602/34750 [9:04:00<25:15:51, 3.62s/it] 28%|██▊ | 9603/34750 [9:04:04<24:41:44, 3.54s/it] 28%|██▊ | 9604/34750 [9:04:07<23:55:48, 3.43s/it] 28%|██▊ | 9605/34750 [9:04:10<23:41:35, 3.39s/it] 28%|██▊ | 9606/34750 [9:04:13<23:17:00, 3.33s/it] 28%|██▊ | 9607/34750 [9:04:16<22:58:05, 3.29s/it] 28%|██▊ | 9608/34750 [9:04:20<22:36:45, 3.24s/it] 28%|██▊ | 9609/34750 [9:04:23<22:27:01, 3.21s/it] 28%|██▊ | 9610/34750 [9:04:26<21:56:13, 3.14s/it] 28%|██▊ | 9611/34750 [9:04:29<21:33:01, 3.09s/it] 28%|██▊ | 9612/34750 [9:04:32<21:18:09, 3.05s/it] 28%|██▊ | 9613/34750 [9:04:34<20:57:31, 3.00s/it] 28%|██▊ | 9614/34750 [9:04:37<20:47:00, 2.98s/it] 28%|██▊ | 9615/34750 [9:04:40<20:19:06, 2.91s/it] 28%|██▊ | 9616/34750 [9:04:43<19:57:47, 2.86s/it] 28%|██▊ | 9617/34750 [9:04:46<19:28:36, 2.79s/it] 28%|██▊ | 9618/34750 [9:04:48<19:13:11, 2.75s/it] 28%|██▊ | 9619/34750 [9:04:51<19:07:49, 2.74s/it] 28%|██▊ | 9620/34750 [9:04:53<18:44:14, 2.68s/it] 28%|██▊ | 9621/34750 [9:04:56<18:32:02, 2.66s/it] 28%|██▊ | 9622/34750 [9:04:59<18:23:07, 2.63s/it] 28%|██▊ | 9623/34750 [9:05:01<18:11:44, 2.61s/it] 28%|██▊ | 9624/34750 [9:05:03<17:29:14, 2.51s/it] 28%|██▊ | 9625/34750 [9:05:06<17:19:20, 2.48s/it] 28%|██▊ | 9626/34750 [9:05:08<16:56:04, 2.43s/it] 28%|██▊ | 9627/34750 [9:05:10<16:37:08, 2.38s/it] 28%|██▊ | 9628/34750 [9:05:13<16:06:34, 2.31s/it] 28%|██▊ | 9629/34750 [9:05:15<15:37:33, 2.24s/it] 28%|██▊ | 9630/34750 [9:05:17<15:16:05, 2.19s/it] 28%|██▊ | 9631/34750 [9:05:19<14:51:50, 2.13s/it] 28%|██▊ | 9632/34750 [9:05:21<14:32:32, 2.08s/it] 28%|██▊ | 9633/34750 [9:05:23<14:08:44, 2.03s/it] 28%|██▊ | 9634/34750 [9:05:24<13:33:59, 1.94s/it] 28%|██▊ | 9635/34750 [9:05:26<13:07:18, 1.88s/it] 28%|██▊ | 9636/34750 [9:05:32<22:33:03, 3.23s/it] 28%|██▊ | 9637/34750 [9:05:38<27:16:35, 3.91s/it] 28%|██▊ | 9638/34750 [9:05:43<30:14:36, 4.34s/it] 28%|██▊ | 9639/34750 [9:05:48<31:28:35, 4.51s/it] 28%|██▊ | 9640/34750 [9:05:53<31:43:53, 4.55s/it] 28%|██▊ | 9641/34750 [9:05:57<31:34:44, 4.53s/it] 28%|██▊ | 9642/34750 [9:06:02<31:29:08, 4.51s/it] 28%|██▊ | 9643/34750 [9:06:06<30:51:48, 4.43s/it] 28%|██▊ | 9644/34750 [9:06:10<30:07:11, 4.32s/it] 28%|██▊ | 9645/34750 [9:06:14<29:29:43, 4.23s/it] 28%|██▊ | 9646/34750 [9:06:18<29:02:00, 4.16s/it] 28%|██▊ | 9647/34750 [9:06:22<28:57:45, 4.15s/it] 28%|██▊ | 9648/34750 [9:06:26<27:45:17, 3.98s/it] 28%|██▊ | 9649/34750 [9:06:30<27:25:17, 3.93s/it] 28%|██▊ | 9650/34750 [9:06:33<26:36:40, 3.82s/it] 28%|██▊ | 9651/34750 [9:06:37<26:19:03, 3.77s/it] 28%|██▊ | 9652/34750 [9:06:40<25:41:33, 3.69s/it] 28%|██▊ | 9653/34750 [9:06:44<25:16:06, 3.62s/it] 28%|██▊ | 9654/34750 [9:06:47<24:44:35, 3.55s/it] 28%|██▊ | 9655/34750 [9:06:50<24:10:58, 3.47s/it] 28%|██▊ | 9656/34750 [9:06:54<23:37:38, 3.39s/it] 28%|██▊ | 9657/34750 [9:06:57<23:31:29, 3.38s/it] 28%|██▊ | 9658/34750 [9:07:00<22:52:34, 3.28s/it] 28%|██▊ | 9659/34750 [9:07:03<22:22:54, 3.21s/it] 28%|██▊ | 9660/34750 [9:07:06<22:05:17, 3.17s/it] 28%|██▊ | 9661/34750 [9:07:09<21:29:16, 3.08s/it] 28%|██▊ | 9662/34750 [9:07:12<21:06:33, 3.03s/it] 28%|██▊ | 9663/34750 [9:07:15<21:01:59, 3.02s/it] 28%|██▊ | 9664/34750 [9:07:18<20:42:52, 2.97s/it] 28%|██▊ | 9665/34750 [9:07:21<20:42:52, 2.97s/it] 28%|██▊ | 9666/34750 [9:07:24<20:26:04, 2.93s/it] 28%|██▊ | 9667/34750 [9:07:26<19:51:59, 2.85s/it] 28%|██▊ | 9668/34750 [9:07:29<19:18:49, 2.77s/it] 28%|██▊ | 9669/34750 [9:07:32<19:02:07, 2.73s/it] 28%|██▊ | 9670/34750 [9:07:34<18:48:17, 2.70s/it] 28%|██▊ | 9671/34750 [9:07:37<18:48:28, 2.70s/it] 28%|██▊ | 9672/34750 [9:07:39<18:23:55, 2.64s/it] 28%|██▊ | 9673/34750 [9:07:42<18:04:03, 2.59s/it] 28%|██▊ | 9674/34750 [9:07:44<17:50:50, 2.56s/it] 28%|██▊ | 9675/34750 [9:07:47<17:20:22, 2.49s/it] 28%|██▊ | 9676/34750 [9:07:49<16:53:46, 2.43s/it] 28%|██▊ | 9677/34750 [9:07:51<16:25:53, 2.36s/it] 28%|██▊ | 9678/34750 [9:07:53<16:03:33, 2.31s/it] 28%|██▊ | 9679/34750 [9:07:55<15:31:08, 2.23s/it] 28%|██▊ | 9680/34750 [9:07:58<15:21:04, 2.20s/it] 28%|██▊ | 9681/34750 [9:08:00<15:12:38, 2.18s/it] 28%|██▊ | 9682/34750 [9:08:02<14:32:35, 2.09s/it] 28%|██▊ | 9683/34750 [9:08:03<14:13:24, 2.04s/it] 28%|██▊ | 9684/34750 [9:08:06<14:12:27, 2.04s/it] 28%|██▊ | 9685/34750 [9:08:07<13:16:24, 1.91s/it] 28%|██▊ | 9686/34750 [9:08:14<23:22:41, 3.36s/it] 28%|██▊ | 9687/34750 [9:08:20<28:30:11, 4.09s/it] 28%|██▊ | 9688/34750 [9:08:25<31:34:54, 4.54s/it] 28%|██▊ | 9689/34750 [9:08:30<32:47:06, 4.71s/it] 28%|██▊ | 9690/34750 [9:08:35<32:38:54, 4.69s/it] 28%|██▊ | 9691/34750 [9:08:39<32:14:34, 4.63s/it] 28%|██▊ | 9692/34750 [9:08:44<31:41:22, 4.55s/it] 28%|██▊ | 9693/34750 [9:08:48<31:34:06, 4.54s/it] 28%|██▊ | 9694/34750 [9:08:53<31:02:29, 4.46s/it] 28%|██▊ | 9695/34750 [9:08:57<29:55:04, 4.30s/it] 28%|██▊ | 9696/34750 [9:09:01<29:34:35, 4.25s/it] 28%|██▊ | 9697/34750 [9:09:05<28:49:59, 4.14s/it] 28%|██▊ | 9698/34750 [9:09:08<27:52:02, 4.00s/it] 28%|██▊ | 9699/34750 [9:09:12<26:46:27, 3.85s/it] 28%|██▊ | 9700/34750 [9:09:15<26:06:31, 3.75s/it] 28%|██▊ | 9700/34750 [9:09:15<26:06:31, 3.75s/it] 28%|██▊ | 9701/34750 [9:09:19<25:38:31, 3.69s/it] 28%|██▊ | 9702/34750 [9:09:22<24:58:56, 3.59s/it] 28%|██▊ | 9703/34750 [9:09:26<24:24:35, 3.51s/it] 28%|██▊ | 9704/34750 [9:09:29<23:30:34, 3.38s/it] 28%|██▊ | 9705/34750 [9:09:32<23:02:09, 3.31s/it] 28%|██▊ | 9706/34750 [9:09:35<22:30:32, 3.24s/it] 28%|██▊ | 9707/34750 [9:09:38<22:21:55, 3.22s/it] 28%|██▊ | 9708/34750 [9:09:41<21:48:58, 3.14s/it] 28%|██▊ | 9709/34750 [9:09:44<21:16:31, 3.06s/it] 28%|██▊ | 9710/34750 [9:09:47<20:58:40, 3.02s/it] 28%|██▊ | 9711/34750 [9:09:49<20:27:57, 2.94s/it] 28%|██▊ | 9712/34750 [9:09:52<19:46:11, 2.84s/it] 28%|██▊ | 9713/34750 [9:09:55<19:17:32, 2.77s/it] 28%|██▊ | 9714/34750 [9:09:57<18:49:42, 2.71s/it] 28%|██▊ | 9715/34750 [9:10:00<19:00:15, 2.73s/it] 28%|██▊ | 9716/34750 [9:10:03<18:33:35, 2.67s/it] 28%|██▊ | 9717/34750 [9:10:05<18:02:14, 2.59s/it] 28%|██▊ | 9718/34750 [9:10:08<18:05:08, 2.60s/it] 28%|██▊ | 9719/34750 [9:10:10<17:30:14, 2.52s/it] 28%|██▊ | 9720/34750 [9:10:12<17:13:42, 2.48s/it] 28%|██▊ | 9721/34750 [9:10:15<16:52:59, 2.43s/it] 28%|██▊ | 9722/34750 [9:10:17<16:42:17, 2.40s/it] 28%|██▊ | 9723/34750 [9:10:19<16:15:26, 2.34s/it] 28%|██▊ | 9724/34750 [9:10:21<15:44:40, 2.26s/it] 28%|██▊ | 9725/34750 [9:10:23<15:25:12, 2.22s/it] 28%|██▊ | 9726/34750 [9:10:25<15:04:03, 2.17s/it] 28%|██▊ | 9727/34750 [9:10:27<14:37:39, 2.10s/it] 28%|██▊ | 9728/34750 [9:10:29<14:11:43, 2.04s/it] 28%|██▊ | 9729/34750 [9:10:31<13:49:27, 1.99s/it] 28%|██▊ | 9730/34750 [9:10:33<13:27:15, 1.94s/it] 28%|██▊ | 9731/34750 [9:10:40<25:11:21, 3.62s/it] 28%|██▊ | 9732/34750 [9:10:46<29:38:44, 4.27s/it] 28%|██▊ | 9733/34750 [9:10:51<31:28:22, 4.53s/it] 28%|██▊ | 9734/34750 [9:10:56<32:04:09, 4.62s/it] 28%|██▊ | 9735/34750 [9:11:01<32:24:33, 4.66s/it] 28%|██▊ | 9736/34750 [9:11:06<32:46:54, 4.72s/it] 28%|██▊ | 9737/34750 [9:11:10<31:33:54, 4.54s/it] 28%|██▊ | 9738/34750 [9:11:14<30:31:30, 4.39s/it] 28%|██▊ | 9739/34750 [9:11:18<29:39:31, 4.27s/it] 28%|██▊ | 9740/34750 [9:11:22<29:01:51, 4.18s/it] 28%|██▊ | 9741/34750 [9:11:26<28:18:46, 4.08s/it] 28%|██▊ | 9742/34750 [9:11:30<27:49:29, 4.01s/it] 28%|██▊ | 9743/34750 [9:11:34<27:50:57, 4.01s/it] 28%|██▊ | 9744/34750 [9:11:37<27:15:59, 3.93s/it] 28%|██▊ | 9745/34750 [9:11:41<26:40:17, 3.84s/it] 28%|██▊ | 9746/34750 [9:11:45<26:09:17, 3.77s/it] 28%|██▊ | 9747/34750 [9:11:48<25:28:38, 3.67s/it] 28%|██▊ | 9748/34750 [9:11:51<24:47:13, 3.57s/it] 28%|██▊ | 9749/34750 [9:11:55<24:08:55, 3.48s/it] 28%|██▊ | 9750/34750 [9:11:58<23:50:34, 3.43s/it] 28%|██▊ | 9751/34750 [9:12:01<23:22:20, 3.37s/it] 28%|██▊ | 9752/34750 [9:12:04<22:49:43, 3.29s/it] 28%|██▊ | 9753/34750 [9:12:07<22:31:09, 3.24s/it] 28%|██▊ | 9754/34750 [9:12:11<22:39:45, 3.26s/it] 28%|██▊ | 9755/34750 [9:12:14<22:19:15, 3.21s/it] 28%|██▊ | 9756/34750 [9:12:17<21:49:25, 3.14s/it] 28%|██▊ | 9757/34750 [9:12:20<21:10:55, 3.05s/it] 28%|██▊ | 9758/34750 [9:12:23<20:45:52, 2.99s/it] 28%|██▊ | 9759/34750 [9:12:25<20:19:07, 2.93s/it] 28%|██▊ | 9760/34750 [9:12:28<20:00:01, 2.88s/it] 28%|██▊ | 9761/34750 [9:12:31<19:49:45, 2.86s/it] 28%|██▊ | 9762/34750 [9:12:34<19:26:02, 2.80s/it] 28%|██▊ | 9763/34750 [9:12:36<18:56:07, 2.73s/it] 28%|██▊ | 9764/34750 [9:12:39<18:42:41, 2.70s/it] 28%|██▊ | 9765/34750 [9:12:41<18:30:35, 2.67s/it] 28%|██▊ | 9766/34750 [9:12:44<18:11:03, 2.62s/it] 28%|██▊ | 9767/34750 [9:12:46<18:04:55, 2.61s/it] 28%|██▊ | 9768/34750 [9:12:49<18:02:37, 2.60s/it] 28%|██▊ | 9769/34750 [9:12:51<17:48:54, 2.57s/it] 28%|██▊ | 9770/34750 [9:12:54<17:10:56, 2.48s/it] 28%|██▊ | 9771/34750 [9:12:56<16:43:22, 2.41s/it] 28%|██▊ | 9772/34750 [9:12:58<16:19:41, 2.35s/it] 28%|██▊ | 9773/34750 [9:13:00<15:55:24, 2.30s/it] 28%|██▊ | 9774/34750 [9:13:03<15:40:11, 2.26s/it] 28%|██▊ | 9775/34750 [9:13:05<15:18:57, 2.21s/it] 28%|██▊ | 9776/34750 [9:13:07<14:48:13, 2.13s/it] 28%|██▊ | 9777/34750 [9:13:09<14:21:32, 2.07s/it] 28%|██▊ | 9778/34750 [9:13:10<13:54:33, 2.01s/it] 28%|██▊ | 9779/34750 [9:13:12<13:20:51, 1.92s/it] 28%|██▊ | 9780/34750 [9:13:14<12:43:33, 1.83s/it] 28%|██▊ | 9781/34750 [9:13:20<22:25:46, 3.23s/it] 28%|██▊ | 9782/34750 [9:13:26<27:26:02, 3.96s/it] 28%|██▊ | 9783/34750 [9:13:31<30:27:48, 4.39s/it] 28%|██▊ | 9784/34750 [9:13:36<32:02:27, 4.62s/it] 28%|██▊ | 9785/34750 [9:13:41<32:20:28, 4.66s/it] 28%|██▊ | 9786/34750 [9:13:46<32:03:33, 4.62s/it] 28%|██▊ | 9787/34750 [9:13:50<31:04:12, 4.48s/it] 28%|██▊ | 9788/34750 [9:13:54<30:36:02, 4.41s/it] 28%|██▊ | 9789/34750 [9:13:58<30:23:25, 4.38s/it] 28%|██▊ | 9790/34750 [9:14:02<29:30:27, 4.26s/it] 28%|██▊ | 9791/34750 [9:14:06<28:33:09, 4.12s/it] 28%|██▊ | 9792/34750 [9:14:10<27:39:05, 3.99s/it] 28%|██▊ | 9793/34750 [9:14:14<26:55:53, 3.88s/it] 28%|██▊ | 9794/34750 [9:14:17<26:24:21, 3.81s/it] 28%|██▊ | 9795/34750 [9:14:21<26:10:55, 3.78s/it] 28%|██▊ | 9796/34750 [9:14:24<25:30:49, 3.68s/it] 28%|██▊ | 9797/34750 [9:14:28<24:51:05, 3.59s/it] 28%|██▊ | 9798/34750 [9:14:31<24:29:08, 3.53s/it] 28%|██▊ | 9799/34750 [9:14:35<24:14:39, 3.50s/it] 28%|██▊ | 9800/34750 [9:14:38<23:42:40, 3.42s/it] 28%|██▊ | 9800/34750 [9:14:38<23:42:40, 3.42s/it] 28%|██▊ | 9801/34750 [9:14:41<23:24:17, 3.38s/it] 28%|██▊ | 9802/34750 [9:14:44<22:45:34, 3.28s/it] 28%|██▊ | 9803/34750 [9:14:47<22:15:57, 3.21s/it] 28%|██▊ | 9804/34750 [9:14:50<22:09:16, 3.20s/it] 28%|██▊ | 9805/34750 [9:14:53<21:51:05, 3.15s/it] 28%|██▊ | 9806/34750 [9:14:56<21:29:56, 3.10s/it] 28%|██▊ | 9807/34750 [9:14:59<20:58:22, 3.03s/it] 28%|██▊ | 9808/34750 [9:15:02<20:30:56, 2.96s/it] 28%|██▊ | 9809/34750 [9:15:05<20:40:51, 2.99s/it] 28%|██▊ | 9810/34750 [9:15:08<20:16:16, 2.93s/it] 28%|██▊ | 9811/34750 [9:15:11<19:55:07, 2.88s/it] 28%|██▊ | 9812/34750 [9:15:13<19:38:21, 2.84s/it] 28%|██▊ | 9813/34750 [9:15:16<19:38:34, 2.84s/it] 28%|██▊ | 9814/34750 [9:15:19<19:13:05, 2.77s/it] 28%|██▊ | 9815/34750 [9:15:21<18:59:27, 2.74s/it] 28%|██▊ | 9816/34750 [9:15:24<18:35:53, 2.69s/it] 28%|██▊ | 9817/34750 [9:15:27<18:22:09, 2.65s/it] 28%|██▊ | 9818/34750 [9:15:29<17:34:50, 2.54s/it] 28%|██▊ | 9819/34750 [9:15:31<17:00:03, 2.45s/it] 28%|██▊ | 9820/34750 [9:15:33<16:33:58, 2.39s/it] 28%|██▊ | 9821/34750 [9:15:36<16:11:09, 2.34s/it] 28%|██▊ | 9822/34750 [9:15:38<15:55:57, 2.30s/it] 28%|██▊ | 9823/34750 [9:15:40<15:43:47, 2.27s/it] 28%|██▊ | 9824/34750 [9:15:42<15:34:17, 2.25s/it] 28%|██▊ | 9825/34750 [9:15:44<15:22:38, 2.22s/it] 28%|██▊ | 9826/34750 [9:15:46<15:10:23, 2.19s/it] 28%|██▊ | 9827/34750 [9:15:49<14:48:57, 2.14s/it] 28%|██▊ | 9828/34750 [9:15:51<14:37:17, 2.11s/it] 28%|██▊ | 9829/34750 [9:15:52<13:53:06, 2.01s/it] 28%|██▊ | 9830/34750 [9:15:54<13:06:48, 1.89s/it] 28%|██▊ | 9831/34750 [9:16:01<23:17:56, 3.37s/it] 28%|██▊ | 9832/34750 [9:16:07<28:16:16, 4.08s/it] 28%|██▊ | 9833/34750 [9:16:12<31:15:59, 4.52s/it] 28%|██▊ | 9834/34750 [9:16:17<32:03:04, 4.63s/it] 28%|██▊ | 9835/34750 [9:16:22<32:19:22, 4.67s/it] 28%|██▊ | 9836/34750 [9:16:26<32:29:42, 4.70s/it] 28%|██▊ | 9837/34750 [9:16:31<31:58:29, 4.62s/it] 28%|██▊ | 9838/34750 [9:16:35<31:19:21, 4.53s/it] 28%|██▊ | 9839/34750 [9:16:39<30:06:46, 4.35s/it] 28%|██▊ | 9840/34750 [9:16:43<29:16:20, 4.23s/it] 28%|██▊ | 9841/34750 [9:16:47<28:50:40, 4.17s/it] 28%|██▊ | 9842/34750 [9:16:51<28:09:28, 4.07s/it] 28%|██▊ | 9843/34750 [9:16:55<27:41:16, 4.00s/it] 28%|██▊ | 9844/34750 [9:16:59<27:08:37, 3.92s/it] 28%|██▊ | 9845/34750 [9:17:02<26:34:55, 3.84s/it] 28%|██▊ | 9846/34750 [9:17:06<25:56:41, 3.75s/it] 28%|██▊ | 9847/34750 [9:17:09<25:35:45, 3.70s/it] 28%|██▊ | 9848/34750 [9:17:13<25:12:25, 3.64s/it] 28%|██▊ | 9849/34750 [9:17:16<25:00:57, 3.62s/it] 28%|██▊ | 9850/34750 [9:17:20<24:23:51, 3.53s/it] 28%|██▊ | 9851/34750 [9:17:23<23:47:45, 3.44s/it] 28%|██▊ | 9852/34750 [9:17:26<23:08:14, 3.35s/it] 28%|██▊ | 9853/34750 [9:17:29<22:36:33, 3.27s/it] 28%|██▊ | 9854/34750 [9:17:32<22:15:24, 3.22s/it] 28%|██▊ | 9855/34750 [9:17:35<21:59:43, 3.18s/it] 28%|██▊ | 9856/34750 [9:17:38<21:47:09, 3.15s/it] 28%|██▊ | 9857/34750 [9:17:41<21:26:11, 3.10s/it] 28%|██▊ | 9858/34750 [9:17:44<21:10:50, 3.06s/it] 28%|██▊ | 9859/34750 [9:17:47<20:41:49, 2.99s/it] 28%|██▊ | 9860/34750 [9:17:50<20:22:35, 2.95s/it] 28%|██▊ | 9861/34750 [9:17:53<20:06:10, 2.91s/it] 28%|██▊ | 9862/34750 [9:17:56<19:42:08, 2.85s/it] 28%|██▊ | 9863/34750 [9:17:58<19:04:48, 2.76s/it] 28%|██▊ | 9864/34750 [9:18:01<18:47:38, 2.72s/it] 28%|██▊ | 9865/34750 [9:18:03<18:23:20, 2.66s/it] 28%|██▊ | 9866/34750 [9:18:06<17:57:34, 2.60s/it] 28%|██▊ | 9867/34750 [9:18:08<17:42:53, 2.56s/it] 28%|██▊ | 9868/34750 [9:18:10<17:08:33, 2.48s/it] 28%|██▊ | 9869/34750 [9:18:13<16:53:39, 2.44s/it] 28%|██▊ | 9870/34750 [9:18:15<16:43:35, 2.42s/it] 28%|██▊ | 9871/34750 [9:18:17<16:12:49, 2.35s/it] 28%|██▊ | 9872/34750 [9:18:20<15:51:13, 2.29s/it] 28%|██▊ | 9873/34750 [9:18:22<15:35:59, 2.26s/it] 28%|██▊ | 9874/34750 [9:18:24<15:21:31, 2.22s/it] 28%|██▊ | 9875/34750 [9:18:26<15:03:43, 2.18s/it] 28%|██▊ | 9876/34750 [9:18:28<14:51:32, 2.15s/it] 28%|██▊ | 9877/34750 [9:18:30<14:22:44, 2.08s/it] 28%|██▊ | 9878/34750 [9:18:32<13:59:31, 2.03s/it] 28%|██▊ | 9879/34750 [9:18:34<13:44:28, 1.99s/it] 28%|██▊ | 9880/34750 [9:18:35<13:06:20, 1.90s/it] 28%|██▊ | 9881/34750 [9:18:41<21:30:05, 3.11s/it] 28%|██▊ | 9882/34750 [9:18:47<26:34:28, 3.85s/it] 28%|██▊ | 9883/34750 [9:18:52<29:24:13, 4.26s/it] 28%|██▊ | 9884/34750 [9:18:57<30:30:09, 4.42s/it] 28%|██▊ | 9885/34750 [9:19:01<30:43:39, 4.45s/it] 28%|██▊ | 9886/34750 [9:19:06<30:41:25, 4.44s/it] 28%|██▊ | 9887/34750 [9:19:10<30:24:00, 4.40s/it] 28%|██▊ | 9888/34750 [9:19:14<30:05:31, 4.36s/it] 28%|██▊ | 9889/34750 [9:19:19<29:31:55, 4.28s/it] 28%|██▊ | 9890/34750 [9:19:23<28:52:06, 4.18s/it] 28%|██▊ | 9891/34750 [9:19:26<28:09:14, 4.08s/it] 28%|██▊ | 9892/34750 [9:19:30<27:23:28, 3.97s/it] 28%|██▊ | 9893/34750 [9:19:34<26:44:27, 3.87s/it] 28%|██▊ | 9894/34750 [9:19:37<26:18:33, 3.81s/it] 28%|██▊ | 9895/34750 [9:19:41<25:50:01, 3.74s/it] 28%|██▊ | 9896/34750 [9:19:44<25:16:58, 3.66s/it] 28%|██▊ | 9897/34750 [9:19:48<24:47:22, 3.59s/it] 28%|██▊ | 9898/34750 [9:19:51<24:18:41, 3.52s/it] 28%|██▊ | 9899/34750 [9:19:55<23:58:04, 3.47s/it] 28%|██▊ | 9900/34750 [9:19:58<23:31:58, 3.41s/it] 28%|██▊ | 9900/34750 [9:19:58<23:31:58, 3.41s/it] 28%|██▊ | 9901/34750 [9:20:01<23:13:38, 3.37s/it] 28%|██▊ | 9902/34750 [9:20:04<23:07:58, 3.35s/it] 28%|██▊ | 9903/34750 [9:20:08<22:39:28, 3.28s/it] 29%|██▊ | 9904/34750 [9:20:11<22:07:58, 3.21s/it] 29%|██▊ | 9905/34750 [9:20:14<21:46:46, 3.16s/it] 29%|██▊ | 9906/34750 [9:20:17<21:26:27, 3.11s/it] 29%|██▊ | 9907/34750 [9:20:20<21:10:16, 3.07s/it] 29%|██▊ | 9908/34750 [9:20:22<20:38:47, 2.99s/it] 29%|██▊ | 9909/34750 [9:20:25<20:09:45, 2.92s/it] 29%|██▊ | 9910/34750 [9:20:28<19:52:10, 2.88s/it] 29%|██▊ | 9911/34750 [9:20:31<19:43:51, 2.86s/it] 29%|██▊ | 9912/34750 [9:20:33<19:26:28, 2.82s/it] 29%|██▊ | 9913/34750 [9:20:36<19:15:34, 2.79s/it] 29%|██▊ | 9914/34750 [9:20:39<18:47:14, 2.72s/it] 29%|██▊ | 9915/34750 [9:20:41<18:21:28, 2.66s/it] 29%|██▊ | 9916/34750 [9:20:44<18:02:55, 2.62s/it] 29%|██▊ | 9917/34750 [9:20:46<17:41:27, 2.56s/it] 29%|██▊ | 9918/34750 [9:20:49<17:12:33, 2.49s/it] 29%|██▊ | 9919/34750 [9:20:51<16:51:02, 2.44s/it] 29%|██▊ | 9920/34750 [9:20:53<16:15:36, 2.36s/it] 29%|██▊ | 9921/34750 [9:20:55<15:53:49, 2.30s/it] 29%|██▊ | 9922/34750 [9:20:57<15:35:09, 2.26s/it] 29%|██▊ | 9923/34750 [9:20:59<15:12:20, 2.20s/it] 29%|██▊ | 9924/34750 [9:21:02<14:54:09, 2.16s/it] 29%|██▊ | 9925/34750 [9:21:04<14:39:43, 2.13s/it] 29%|██▊ | 9926/34750 [9:21:06<14:26:15, 2.09s/it] 29%|██▊ | 9927/34750 [9:21:07<14:04:26, 2.04s/it] 29%|██▊ | 9928/34750 [9:21:09<13:45:46, 2.00s/it] 29%|██▊ | 9929/34750 [9:21:11<13:21:03, 1.94s/it] 29%|██▊ | 9930/34750 [9:21:13<12:43:32, 1.85s/it] 29%|██▊ | 9931/34750 [9:21:19<21:40:52, 3.14s/it] 29%|██▊ | 9932/34750 [9:21:25<27:01:50, 3.92s/it] 29%|██▊ | 9933/34750 [9:21:30<29:52:45, 4.33s/it] 29%|██▊ | 9934/34750 [9:21:35<31:19:03, 4.54s/it] 29%|██▊ | 9935/34750 [9:21:40<31:20:45, 4.55s/it] 29%|██▊ | 9936/34750 [9:21:44<31:27:07, 4.56s/it] 29%|██▊ | 9937/34750 [9:21:49<31:02:13, 4.50s/it] 29%|██▊ | 9938/34750 [9:21:53<30:31:36, 4.43s/it] 29%|██▊ | 9939/34750 [9:21:57<29:59:17, 4.35s/it] 29%|██▊ | 9940/34750 [9:22:01<29:47:03, 4.32s/it] 29%|██▊ | 9941/34750 [9:22:05<29:12:10, 4.24s/it] 29%|██▊ | 9942/34750 [9:22:09<28:30:16, 4.14s/it] 29%|██▊ | 9943/34750 [9:22:13<27:48:23, 4.04s/it] 29%|██▊ | 9944/34750 [9:22:17<27:03:18, 3.93s/it] 29%|██▊ | 9945/34750 [9:22:20<26:25:56, 3.84s/it] 29%|██▊ | 9946/34750 [9:22:24<25:57:25, 3.77s/it] 29%|██▊ | 9947/34750 [9:22:27<25:16:58, 3.67s/it] 29%|██▊ | 9948/34750 [9:22:31<24:45:04, 3.59s/it] 29%|██▊ | 9949/34750 [9:22:34<24:27:23, 3.55s/it] 29%|██▊ | 9950/34750 [9:22:38<24:11:42, 3.51s/it] 29%|██▊ | 9951/34750 [9:22:41<23:38:38, 3.43s/it] 29%|██▊ | 9952/34750 [9:22:44<23:21:44, 3.39s/it] 29%|██▊ | 9953/34750 [9:22:47<22:53:22, 3.32s/it] 29%|██▊ | 9954/34750 [9:22:50<22:32:32, 3.27s/it] 29%|██▊ | 9955/34750 [9:22:54<22:22:47, 3.25s/it] 29%|██▊ | 9956/34750 [9:22:57<21:51:21, 3.17s/it] 29%|██▊ | 9957/34750 [9:23:00<21:42:54, 3.15s/it] 29%|██▊ | 9958/34750 [9:23:03<21:38:51, 3.14s/it] 29%|██▊ | 9959/34750 [9:23:06<21:20:46, 3.10s/it] 29%|██▊ | 9960/34750 [9:23:09<20:48:01, 3.02s/it] 29%|██▊ | 9961/34750 [9:23:11<20:02:38, 2.91s/it] 29%|██▊ | 9962/34750 [9:23:14<19:44:26, 2.87s/it] 29%|██▊ | 9963/34750 [9:23:17<19:27:42, 2.83s/it] 29%|██▊ | 9964/34750 [9:23:19<18:51:34, 2.74s/it] 29%|██▊ | 9965/34750 [9:23:22<18:26:39, 2.68s/it] 29%|██▊ | 9966/34750 [9:23:25<18:17:28, 2.66s/it] 29%|██▊ | 9967/34750 [9:23:27<18:00:03, 2.61s/it] 29%|██▊ | 9968/34750 [9:23:30<17:36:28, 2.56s/it] 29%|██▊ | 9969/34750 [9:23:32<17:00:58, 2.47s/it] 29%|██▊ | 9970/34750 [9:23:34<16:32:04, 2.40s/it] 29%|██▊ | 9971/34750 [9:23:36<16:18:18, 2.37s/it] 29%|██▊ | 9972/34750 [9:23:38<15:51:55, 2.31s/it] 29%|██▊ | 9973/34750 [9:23:41<15:51:32, 2.30s/it] 29%|██▊ | 9974/34750 [9:23:43<15:36:07, 2.27s/it] 29%|██▊ | 9975/34750 [9:23:45<15:33:41, 2.26s/it] 29%|██▊ | 9976/34750 [9:23:47<15:21:49, 2.23s/it] 29%|██▊ | 9977/34750 [9:23:49<15:03:47, 2.19s/it] 29%|██▊ | 9978/34750 [9:23:52<14:48:53, 2.15s/it] 29%|██▊ | 9979/34750 [9:23:53<14:15:16, 2.07s/it] 29%|██▊ | 9980/34750 [9:23:55<13:42:01, 1.99s/it] 29%|██▊ | 9981/34750 [9:24:02<23:00:12, 3.34s/it] 29%|██▊ | 9982/34750 [9:24:07<27:37:03, 4.01s/it] 29%|██▊ | 9983/34750 [9:24:13<30:30:19, 4.43s/it] 29%|██▊ | 9984/34750 [9:24:18<32:13:29, 4.68s/it] 29%|██▊ | 9985/34750 [9:24:23<32:27:21, 4.72s/it] 29%|██▊ | 9986/34750 [9:24:28<32:32:32, 4.73s/it] 29%|██▊ | 9987/34750 [9:24:32<31:53:04, 4.64s/it] 29%|██▊ | 9988/34750 [9:24:36<31:05:46, 4.52s/it] 29%|██▊ | 9989/34750 [9:24:40<30:10:40, 4.39s/it] 29%|██▊ | 9990/34750 [9:24:44<29:49:55, 4.34s/it] 29%|██▉ | 9991/34750 [9:24:49<29:18:35, 4.26s/it] 29%|██▉ | 9992/34750 [9:24:52<28:31:57, 4.15s/it] 29%|██▉ | 9993/34750 [9:24:56<27:58:29, 4.07s/it] 29%|██▉ | 9994/34750 [9:25:00<27:17:50, 3.97s/it] 29%|██▉ | 9995/34750 [9:25:04<26:44:14, 3.89s/it] 29%|██▉ | 9996/34750 [9:25:07<26:19:29, 3.83s/it] 29%|██▉ | 9997/34750 [9:25:11<25:28:22, 3.70s/it] 29%|██▉ | 9998/34750 [9:25:14<24:46:06, 3.60s/it] 29%|██▉ | 9999/34750 [9:25:17<24:01:40, 3.49s/it] 29%|██▉ | 10000/34750 [9:25:21<23:33:36, 3.43s/it] 29%|██▉ | 10000/34750 [9:25:21<23:33:36, 3.43s/it]The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length. +***** Running Evaluation ***** + Num examples = 456 + Batch size = 8 +{'eval_loss': 0.6285760402679443, 'eval_wer': 0.6122017517366355, 'eval_cer': 0.21484805542434263, 'eval_runtime': 41.6653, 'eval_samples_per_second': 10.944, 'eval_steps_per_second': 1.368, 'epoch': 13.67} +{'loss': 1.8236, 'learning_rate': 5.760916030534351e-05, 'epoch': 13.81} +{'loss': 1.8199, 'learning_rate': 5.738015267175571e-05, 'epoch': 13.96} +{'loss': 1.8285, 'learning_rate': 5.7151145038167934e-05, 'epoch': 14.1} +{'loss': 1.817, 'learning_rate': 5.692213740458015e-05, 'epoch': 14.24} +{'loss': 1.819, 'learning_rate': 5.669312977099236e-05, 'epoch': 14.39} + + 0%| | 0/57 [00:00