Training in progress, step 1000

errs

@@ -453,4 +453,49 @@ wandb: Run `wandb offline` to turn off syncing.
 Configuration saved in ./checkpoint-500/config.json
 Model weights saved in ./checkpoint-500/pytorch_model.bin
 Configuration saved in ./checkpoint-500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+
  3% 501/18000 [1:20:58<268:47:20, 55.30s/it]
  3% 502/18000 [1:21:08<202:41:14, 41.70s/it]
  3% 503/18000 [1:21:15<152:19:33, 31.34s/it]
  3% 504/18000 [1:21:18<111:17:11, 22.90s/it]
  3% 505/18000 [1:21:35<102:31:52, 21.10s/it]
  3% 506/18000 [1:21:49<91:33:15, 18.84s/it] 
  3% 507/18000 [1:22:00<80:11:06, 16.50s/it]
  3% 508/18000 [1:22:09<69:47:22, 14.36s/it]
  3% 509/18000 [1:22:18<61:26:47, 12.65s/it]
  3% 510/18000 [1:22:25<53:34:10, 11.03s/it]
  3% 511/18000 [1:22:31<46:14:01,  9.52s/it]
  3% 512/18000 [1:22:36<38:53:55,  8.01s/it]
  3% 513/18000 [1:22:52<50:23:49, 10.38s/it]
  3% 514/18000 [1:23:05<55:08:27, 11.35s/it]
  3% 515/18000 [1:23:16<54:37:40, 11.25s/it]
  3% 516/18000 [1:23:26<51:49:50, 10.67s/it]
  3% 517/18000 [1:23:34<48:28:05,  9.98s/it]
  3% 518/18000 [1:23:41<43:48:52,  9.02s/it]
  3% 519/18000 [1:23:47<39:13:46,  8.08s/it]
  3% 520/18000 [1:23:51<34:02:15,  7.01s/it]
  3% 521/18000 [1:24:08<48:05:46,  9.91s/it]
  3% 522/18000 [1:24:21<53:02:36, 10.93s/it]
  3% 523/18000 [1:24:32<52:51:19, 10.89s/it]
  3% 524/18000 [1:24:41<50:20:29, 10.37s/it]
  3% 525/18000 [1:24:49<47:20:46,  9.75s/it]
  3% 526/18000 [1:24:56<43:24:06,  8.94s/it]
  3% 527/18000 [1:25:02<38:47:19,  7.99s/it]
  3% 528/18000 [1:25:07<33:35:16,  6.92s/it]
  3% 529/18000 [1:25:23<47:37:33,  9.81s/it]
  3% 530/18000 [1:25:37<52:51:32, 10.89s/it]
  3% 531/18000 [1:25:48<52:52:41, 10.90s/it]
  3% 532/18000 [1:25:57<50:26:24, 10.40s/it]
  3% 533/18000 [1:26:05<46:45:36,  9.64s/it]
  3% 534/18000 [1:26:11<42:26:58,  8.75s/it]
  3% 535/18000 [1:26:17<37:58:08,  7.83s/it]
  3% 536/18000 [1:26:21<33:03:22,  6.81s/it]
  3% 537/18000 [1:26:36<44:56:50,  9.27s/it]
  3% 538/18000 [1:26:46<45:20:51,  9.35s/it]
  3% 539/18000 [1:26:53<42:25:38,  8.75s/it]
  3% 540/18000 [1:26:57<34:28:31,  7.11s/it]
  3% 541/18000 [1:27:13<47:27:56,  9.79s/it]
  3% 542/18000 [1:27:26<52:44:41, 10.88s/it]
  3% 543/18000 [1:27:37<52:39:43, 10.86s/it]
  3% 544/18000 [1:27:47<51:02:14, 10.53s/it]
  3% 545/18000 [1:27:55<47:44:16,  9.85s/it]
  3% 546/18000 [1:28:02<43:55:54,  9.06s/it]
  3% 547/18000 [1:28:08<39:28:56,  8.14s/it]
  3% 548/18000 [1:28:13<34:16:05,  7.07s/it]
  3% 549/18000 [1:28:29<48:13:46,  9.95s/it]
  3% 550/18000 [1:28:43<53:26:02, 11.02s/it]
  3% 551/18000 [1:28:53<52:48:01, 10.89s/it]
  3% 552/18000 [1:29:03<50:24:41, 10.40s/it]
  3% 553/18000 [1:29:11<47:30:24,  9.80s/it]
  3% 554/18000 [1:29:18<43:19:19,  8.94s/it]
  3% 555/18000 [1:29:24<38:30:04,  7.95s/it]
  3% 556/18000 [1:29:28<33:20:06,  6.88s/it]
  3% 557/18000 [1:29:45<47:24:33,  9.78s/it]
  3% 558/18000 [1:29:58<52:46:13, 10.89s/it]
  3% 559/18000 [1:30:09<52:35:40, 10.86s/it]
  3% 560/18000 [1:30:18<50:14:14, 10.37s/it]
  3% 561/18000 [1:30:26<46:33:57,  9.61s/it]
  3% 562/18000 [1:30:33<42:35:08,  8.79s/it]
  3% 563/18000 [1:30:39<38:09:47,  7.88s/it]
  3% 564/18000 [1:30:43<33:08:02,  6.84s/it]
  3% 565/18000 [1:30:59<47:11:25,  9.74s/it]
  3% 566/18000 [1:31:13<52:22:48, 10.82s/it]
  3% 567/18000 [1:31:24<52:28:43, 10.84s/it]
  3% 568/18000 [1:31:33<50:20:44, 10.40s/it]
  3% 569/18000 [1:31:41<46:56:59,  9.70s/it]
  3% 570/18000 [1:31:48<43:03:19,  8.89s/it]
  3% 571/18000 [1:31:54<38:29:47,  7.95s/it]
  3% 572/18000 [1:31:58<33:18:33,  6.88s/it]
  3% 573/18000 [1:32:13<45:17:23,  9.36s/it]
  3% 574/18000 [1:32:23<46:16:20,  9.56s/it]
  3% 575/18000 [1:32:31<42:54:39,  8.87s/it]
  3% 576/18000 [1:32:34<34:57:45,  7.22s/it]
  3% 577/18000 [1:32:51<48:43:03, 10.07s/it]
  3% 578/18000 [1:33:04<53:38:53, 11.09s/it]
  3% 579/18000 [1:33:15<53:17:15, 11.01s/it]
  3% 580/18000 [1:33:24<50:56:43, 10.53s/it]
  3% 581/18000 [1:33:33<48:05:25,  9.94s/it]
  3% 582/18000 [1:33:40<44:02:26,  9.10s/it]
  3% 583/18000 [1:33:46<39:24:10,  8.14s/it]
  3% 584/18000 [1:33:51<33:59:21,  7.03s/it]
  3% 585/18000 [1:34:07<47:30:53,  9.82s/it]
  3% 586/18000 [1:34:20<52:28:42, 10.85s/it]
  3% 587/18000 [1:34:31<52:17:26, 10.81s/it]
  3% 588/18000 [1:34:40<50:10:45, 10.37s/it]
  3% 589/18000 [1:34:49<47:16:06,  9.77s/it]
  3% 590/18000 [1:34:55<43:06:03,  8.91s/it]
  3% 591/18000 [1:35:01<38:35:48,  7.98s/it]
  3% 592/18000 [1:35:06<33:22:46,  6.90s/it]
  3% 593/18000 [1:35:22<47:13:11,  9.77s/it]
  3% 594/18000 [1:35:35<52:26:16, 10.85s/it]
  3% 595/18000 [1:35:46<52:34:27, 10.87s/it]
  3% 596/18000 [1:35:56<50:30:36, 10.45s/it]
  3% 597/18000 [1:36:04<47:20:14,  9.79s/it]
  3% 598/18000 [1:36:11<43:19:34,  8.96s/it]
  3% 599/18000 [1:36:17<38:45:13,  8.02s/it]
  3% 600/18000 [1:36:22<33:47:29,  6.99s/it]
                                            
  3% 600/18000 [1:36:22<33:47:29,  6.99s/it]
  3% 601/18000 [1:36:38<47:50:06,  9.90s/it]
  3% 602/18000 [1:36:52<53:11:56, 11.01s/it]
  3% 603/18000 [1:37:03<53:09:51, 11.00s/it]
  3% 604/18000 [1:37:12<50:30:53, 10.45s/it]
  3% 605/18000 [1:37:20<47:23:30,  9.81s/it]
  3% 606/18000 [1:37:27<43:21:26,  8.97s/it]
  3% 607/18000 [1:37:33<38:54:48,  8.05s/it]
  3% 608/18000 [1:37:38<33:50:11,  7.00s/it]
  3% 609/18000 [1:37:53<45:50:17,  9.49s/it]
  3% 610/18000 [1:38:02<45:25:57,  9.41s/it]
  3% 611/18000 [1:38:09<41:59:49,  8.69s/it]
  3% 612/18000 [1:38:13<34:01:51,  7.05s/it]
  3% 613/18000 [1:38:29<47:43:38,  9.88s/it]
  3% 614/18000 [1:38:42<52:10:36, 10.80s/it]
  3% 615/18000 [1:38:53<52:32:10, 10.88s/it]
  3% 616/18000 [1:39:02<50:24:39, 10.44s/it]
  3% 617/18000 [1:39:11<47:25:52,  9.82s/it]
  3% 618/18000 [1:39:18<43:25:05,  8.99s/it]
  3% 619/18000 [1:39:24<38:54:04,  8.06s/it]
  3% 620/18000 [1:39:28<33:42:51,  6.98s/it]
  3% 621/18000 [1:39:45<47:23:18,  9.82s/it]
  3% 622/18000 [1:39:57<51:39:22, 10.70s/it]
  3% 623/18000 [1:40:08<51:27:43, 10.66s/it]
  3% 624/18000 [1:40:17<49:29:02, 10.25s/it]
  3% 625/18000 [1:40:25<46:25:08,  9.62s/it]
  3% 626/18000 [1:40:32<42:30:13,  8.81s/it]
  3% 627/18000 [1:40:38<37:53:11,  7.85s/it]
  3% 628/18000 [1:40:42<32:53:47,  6.82s/it]
  3% 629/18000 [1:40:59<47:24:56,  9.83s/it]
  4% 630/18000 [1:41:13<52:50:43, 10.95s/it]
  4% 631/18000 [1:41:24<52:37:01, 10.91s/it]
  4% 632/18000 [1:41:33<50:29:45, 10.47s/it]
  4% 633/18000 [1:41:42<47:37:06,  9.87s/it]
  4% 634/18000 [1:41:48<43:24:29,  9.00s/it]
  4% 635/18000 [1:41:54<38:50:54,  8.05s/it]
  4% 636/18000 [1:41:59<33:36:27,  6.97s/it]
  4% 637/18000 [1:42:15<47:27:07,  9.84s/it]
  4% 638/18000 [1:42:29<52:35:41, 10.91s/it]
  4% 639/18000 [1:42:40<52:45:13, 10.94s/it]
  4% 640/18000 [1:42:49<50:58:07, 10.57s/it]
  4% 641/18000 [1:42:58<47:47:05,  9.91s/it]
  4% 642/18000 [1:43:05<43:22:50,  9.00s/it]
  4% 643/18000 [1:43:10<38:47:50,  8.05s/it]
  4% 644/18000 [1:43:15<33:37:43,  6.98s/it]
  4% 645/18000 [1:43:30<45:45:05,  9.49s/it]
  4% 646/18000 [1:43:40<46:13:59,  9.59s/it]
  4% 647/18000 [1:43:47<42:45:24,  8.87s/it]
  4% 648/18000 [1:43:51<34:33:47,  7.17s/it]
  4% 649/18000 [1:44:07<48:22:16, 10.04s/it]
  4% 650/18000 [1:44:21<53:33:32, 11.11s/it]
  4% 651/18000 [1:44:32<53:13:33, 11.04s/it]
  4% 652/18000 [1:44:41<50:31:20, 10.48s/it]
  4% 653/18000 [1:44:49<47:17:23,  9.81s/it]
  4% 654/18000 [1:44:56<43:05:08,  8.94s/it]
  4% 655/18000 [1:45:02<38:17:23,  7.95s/it]
  4% 656/18000 [1:45:06<33:11:31,  6.89s/it]
  4% 657/18000 [1:45:23<47:09:08,  9.79s/it]
  4% 658/18000 [1:45:36<52:01:53, 10.80s/it]
  4% 659/18000 [1:45:46<51:15:47, 10.64s/it]
  4% 660/18000 [1:45:55<49:19:57, 10.24s/it]
  4% 661/18000 [1:46:04<46:19:36,  9.62s/it]
  4% 662/18000 [1:46:11<42:29:44,  8.82s/it]
  4% 663/18000 [1:46:16<38:05:42,  7.91s/it]
  4% 664/18000 [1:46:21<33:05:24,  6.87s/it]
  4% 665/18000 [1:46:37<47:05:01,  9.78s/it]
  4% 666/18000 [1:46:51<52:11:00, 10.84s/it]
  4% 667/18000 [1:47:01<52:05:33, 10.82s/it]
  4% 668/18000 [1:47:11<50:01:41, 10.39s/it]
  4% 669/18000 [1:47:19<46:55:28,  9.75s/it]
  4% 670/18000 [1:47:26<43:00:02,  8.93s/it]
  4% 671/18000 [1:47:32<38:29:31,  8.00s/it]
  4% 672/18000 [1:47:36<33:17:11,  6.92s/it]
  4% 673/18000 [1:47:53<47:11:55,  9.81s/it]
  4% 674/18000 [1:48:06<52:34:01, 10.92s/it]
  4% 675/18000 [1:48:17<52:37:30, 10.94s/it]
  4% 676/18000 [1:48:27<50:57:47, 10.59s/it]
  4% 677/18000 [1:48:36<47:50:56,  9.94s/it]
  4% 678/18000 [1:48:43<43:38:39,  9.07s/it]
  4% 679/18000 [1:48:48<39:00:27,  8.11s/it]
  4% 680/18000 [1:48:53<33:39:27,  7.00s/it]
  4% 681/18000 [1:49:08<45:38:32,  9.49s/it]
  4% 682/18000 [1:49:18<46:33:58,  9.68s/it]
  4% 683/18000 [1:49:26<43:23:42,  9.02s/it]
  4% 684/18000 [1:49:29<35:17:57,  7.34s/it]
  4% 685/18000 [1:49:46<48:46:52, 10.14s/it]
  4% 686/18000 [1:49:59<53:23:08, 11.10s/it]
  4% 687/18000 [1:50:10<52:48:51, 10.98s/it]
  4% 688/18000 [1:50:19<50:28:17, 10.50s/it]
  4% 689/18000 [1:50:28<47:18:32,  9.84s/it]
  4% 690/18000 [1:50:35<43:11:35,  8.98s/it]
  4% 691/18000 [1:50:40<38:21:06,  7.98s/it]
  4% 692/18000 [1:50:45<33:23:39,  6.95s/it]
  4% 693/18000 [1:51:01<47:15:47,  9.83s/it]
  4% 694/18000 [1:51:15<52:13:35, 10.86s/it]
  4% 695/18000 [1:51:25<52:14:41, 10.87s/it]
  4% 696/18000 [1:51:35<50:08:40, 10.43s/it]
  4% 697/18000 [1:51:43<46:55:02,  9.76s/it]
  4% 698/18000 [1:51:50<42:50:18,  8.91s/it]
  4% 699/18000 [1:51:56<38:15:23,  7.96s/it]
  4% 700/18000 [1:52:00<33:19:23,  6.93s/it]
                                            
  4% 700/18000 [1:52:00<33:19:23,  6.93s/it]
  4% 701/18000 [1:52:17<46:54:47,  9.76s/it]
  4% 702/18000 [1:52:30<51:54:11, 10.80s/it]
  4% 703/18000 [1:52:41<52:06:41, 10.85s/it]
  4% 704/18000 [1:52:50<50:17:18, 10.47s/it]
  4% 705/18000 [1:52:59<46:50:47,  9.75s/it]
  4% 706/18000 [1:53:05<42:45:36,  8.90s/it]
  4% 707/18000 [1:53:11<38:29:11,  8.01s/it]
  4% 708/18000 [1:53:16<33:28:21,  6.97s/it]
  4% 709/18000 [1:53:33<47:27:17,  9.88s/it]
  4% 710/18000 [1:53:46<52:51:37, 11.01s/it]
  4% 711/18000 [1:53:57<52:46:21, 10.99s/it]
  4% 712/18000 [1:54:07<50:54:23, 10.60s/it]
  4% 713/18000 [1:54:15<47:32:14,  9.90s/it]
  4% 714/18000 [1:54:22<43:42:55,  9.10s/it]
  4% 715/18000 [1:54:29<39:27:23,  8.22s/it]
  4% 716/18000 [1:54:33<34:03:58,  7.10s/it]
  4% 717/18000 [1:54:48<44:49:37,  9.34s/it]
  4% 718/18000 [1:54:57<45:09:17,  9.41s/it]
  4% 719/18000 [1:55:04<41:37:38,  8.67s/it]
  4% 720/18000 [1:55:07<33:34:35,  7.00s/it]
  4% 721/18000 [1:55:24<47:24:49,  9.88s/it]
  4% 722/18000 [1:55:37<51:57:23, 10.83s/it]
  4% 723/18000 [1:55:48<51:48:55, 10.80s/it]
  4% 724/18000 [1:55:57<49:52:09, 10.39s/it]
  4% 725/18000 [1:56:05<46:45:48,  9.75s/it]
  4% 726/18000 [1:56:12<42:40:18,  8.89s/it]
  4% 727/18000 [1:56:18<37:54:57,  7.90s/it]
  4% 728/18000 [1:56:22<32:51:09,  6.85s/it]
  4% 729/18000 [1:56:39<46:48:53,  9.76s/it]
  4% 730/18000 [1:56:52<51:53:16, 10.82s/it]
  4% 731/18000 [1:57:03<51:41:09, 10.77s/it]
  4% 732/18000 [1:57:12<49:37:43, 10.35s/it]
  4% 733/18000 [1:57:20<46:39:55,  9.73s/it]
  4% 734/18000 [1:57:27<42:46:04,  8.92s/it]
  4% 735/18000 [1:57:33<38:16:58,  7.98s/it]
  4% 736/18000 [1:57:37<32:59:39,  6.88s/it]
  4% 737/18000 [1:57:54<46:58:28,  9.80s/it]
  4% 738/18000 [1:58:08<52:26:36, 10.94s/it]
  4% 739/18000 [1:58:19<52:40:35, 10.99s/it]
  4% 740/18000 [1:58:28<50:51:29, 10.61s/it]
  4% 741/18000 [1:58:37<47:16:32,  9.86s/it]
  4% 742/18000 [1:58:44<43:13:02,  9.02s/it]
  4% 743/18000 [1:58:49<38:42:01,  8.07s/it]
  4% 744/18000 [1:58:54<33:26:01,  6.98s/it]
  4% 745/18000 [1:59:10<47:18:20,  9.87s/it]
  4% 746/18000 [1:59:24<52:49:28, 11.02s/it]
  4% 747/18000 [1:59:35<52:50:13, 11.02s/it]
  4% 748/18000 [1:59:45<50:45:21, 10.59s/it]
  4% 749/18000 [1:59:53<47:37:50,  9.94s/it]
  4% 750/18000 [2:00:01<43:48:44,  9.14s/it]
  4% 751/18000 [2:00:07<39:19:47,  8.21s/it]
  4% 752/18000 [2:00:11<34:09:01,  7.13s/it]
  4% 753/18000 [2:00:26<45:13:18,  9.44s/it]
  4% 754/18000 [2:00:36<45:37:45,  9.52s/it]
  4% 755/18000 [2:00:43<42:23:05,  8.85s/it]
  4% 756/18000 [2:00:46<34:14:05,  7.15s/it]
  4% 757/18000 [2:01:03<48:03:59, 10.04s/it]
  4% 758/18000 [2:01:16<53:04:06, 11.08s/it]
  4% 759/18000 [2:01:27<51:39:56, 10.79s/it]
  4% 760/18000 [2:01:36<49:23:44, 10.31s/it]
  4% 761/18000 [2:01:44<46:36:38,  9.73s/it]
  4% 762/18000 [2:01:51<42:52:02,  8.95s/it]
  4% 763/18000 [2:01:57<38:20:00,  8.01s/it]
  4% 764/18000 [2:02:01<33:09:23,  6.93s/it]
  4% 765/18000 [2:02:18<46:52:35,  9.79s/it]
  4% 766/18000 [2:02:31<51:55:49, 10.85s/it]
  4% 767/18000 [2:02:42<51:50:49, 10.83s/it]
  4% 768/18000 [2:02:51<49:43:55, 10.39s/it]
  4% 769/18000 [2:03:00<46:39:44,  9.75s/it]
  4% 770/18000 [2:03:06<42:27:00,  8.87s/it]
  4% 771/18000 [2:03:12<38:08:58,  7.97s/it]
  4% 772/18000 [2:03:17<33:03:41,  6.91s/it]
  4% 773/18000 [2:03:33<46:48:19,  9.78s/it]
  4% 774/18000 [2:03:47<51:53:59, 10.85s/it]
  4% 775/18000 [2:03:57<51:47:02, 10.82s/it]
  4% 776/18000 [2:04:07<50:00:40, 10.45s/it]
  4% 777/18000 [2:04:15<47:09:46,  9.86s/it]
  4% 778/18000 [2:04:22<43:05:38,  9.01s/it]
  4% 779/18000 [2:04:28<38:26:00,  8.03s/it]
  4% 780/18000 [2:04:33<33:20:56,  6.97s/it]
  4% 781/18000 [2:04:49<47:09:11,  9.86s/it]
  4% 782/18000 [2:05:03<52:15:07, 10.93s/it]
  4% 783/18000 [2:05:14<52:10:35, 10.91s/it]
  4% 784/18000 [2:05:23<50:15:08, 10.51s/it]
  4% 785/18000 [2:05:31<47:06:00,  9.85s/it]
  4% 786/18000 [2:05:39<43:03:35,  9.01s/it]
  4% 787/18000 [2:05:44<38:32:01,  8.06s/it]
  4% 788/18000 [2:05:49<33:22:00,  6.98s/it]
  4% 789/18000 [2:06:04<45:19:24,  9.48s/it]
  4% 790/18000 [2:06:14<45:50:37,  9.59s/it]
  4% 791/18000 [2:06:21<42:24:51,  8.87s/it]
  4% 792/18000 [2:06:25<34:31:33,  7.22s/it]
  4% 793/18000 [2:06:41<47:56:31, 10.03s/it]
  4% 794/18000 [2:06:55<52:46:41, 11.04s/it]
  4% 795/18000 [2:07:05<52:12:54, 10.93s/it]
  4% 796/18000 [2:07:15<50:55:23, 10.66s/it]
  4% 797/18000 [2:07:23<47:25:26,  9.92s/it]
  4% 798/18000 [2:07:31<43:25:29,  9.09s/it]
  4% 799/18000 [2:07:36<38:49:02,  8.12s/it]
  4% 800/18000 [2:07:41<33:38:43,  7.04s/it]
                                            
  4% 800/18000 [2:07:41<33:38:43,  7.04s/it]
  4% 801/18000 [2:07:57<47:13:37,  9.89s/it]
  4% 802/18000 [2:08:11<52:19:21, 10.95s/it]
  4% 803/18000 [2:08:22<51:54:56, 10.87s/it]
  4% 804/18000 [2:08:31<49:44:27, 10.41s/it]
  4% 805/18000 [2:08:39<46:26:54,  9.72s/it]
  4% 806/18000 [2:08:46<42:21:46,  8.87s/it]
  4% 807/18000 [2:08:52<37:53:34,  7.93s/it]
  4% 808/18000 [2:08:56<32:43:12,  6.85s/it]
  4% 809/18000 [2:09:13<46:42:14,  9.78s/it]
  4% 810/18000 [2:09:26<51:58:37, 10.89s/it]
  5% 811/18000 [2:09:37<51:59:56, 10.89s/it]
  5% 812/18000 [2:09:46<49:34:49, 10.38s/it]
  5% 813/18000 [2:09:55<46:43:44,  9.79s/it]
  5% 814/18000 [2:10:02<42:44:23,  8.95s/it]
  5% 815/18000 [2:10:07<37:59:52,  7.96s/it]
  5% 816/18000 [2:10:12<32:47:42,  6.87s/it]
  5% 817/18000 [2:10:28<46:30:37,  9.74s/it]
  5% 818/18000 [2:10:41<51:22:03, 10.76s/it]
  5% 819/18000 [2:10:52<51:17:55, 10.75s/it]
  5% 820/18000 [2:11:01<49:30:02, 10.37s/it]
  5% 821/18000 [2:11:10<46:19:21,  9.71s/it]
  5% 822/18000 [2:11:17<42:26:40,  8.90s/it]
  5% 823/18000 [2:11:22<38:06:46,  7.99s/it]
  5% 824/18000 [2:11:27<33:12:32,  6.96s/it]
  5% 825/18000 [2:11:42<45:15:06,  9.49s/it]
  5% 826/18000 [2:11:52<46:10:58,  9.68s/it]
  5% 827/18000 [2:12:00<42:58:11,  9.01s/it]
  5% 828/18000 [2:12:03<34:53:39,  7.32s/it]
  5% 829/18000 [2:12:20<48:25:08, 10.15s/it]
  5% 830/18000 [2:12:33<53:03:30, 11.12s/it]
  5% 831/18000 [2:12:44<52:32:04, 11.02s/it]
  5% 832/18000 [2:12:54<50:12:27, 10.53s/it]
  5% 833/18000 [2:13:02<46:51:40,  9.83s/it]
  5% 834/18000 [2:13:09<42:41:55,  8.95s/it]
  5% 835/18000 [2:13:14<37:55:23,  7.95s/it]
  5% 836/18000 [2:13:19<32:41:19,  6.86s/it]
  5% 837/18000 [2:13:35<46:21:35,  9.72s/it]
  5% 838/18000 [2:13:48<51:25:09, 10.79s/it]
  5% 839/18000 [2:13:59<51:04:11, 10.71s/it]
  5% 840/18000 [2:14:08<49:22:23, 10.36s/it]
  5% 841/18000 [2:14:17<46:22:50,  9.73s/it]
  5% 842/18000 [2:14:24<42:26:33,  8.91s/it]
  5% 843/18000 [2:14:29<38:01:14,  7.98s/it]
  5% 844/18000 [2:14:34<32:56:24,  6.91s/it]
  5% 845/18000 [2:14:50<46:30:12,  9.76s/it]
  5% 846/18000 [2:15:04<51:31:14, 10.81s/it]
  5% 847/18000 [2:15:14<51:29:41, 10.81s/it]
  5% 848/18000 [2:15:24<49:34:43, 10.41s/it]
  5% 849/18000 [2:15:32<46:42:35,  9.80s/it]
  5% 850/18000 [2:15:39<42:57:45,  9.02s/it]
  5% 851/18000 [2:15:45<38:31:32,  8.09s/it]
  5% 852/18000 [2:15:50<33:25:05,  7.02s/it]
  5% 853/18000 [2:16:06<46:59:14,  9.86s/it]
  5% 854/18000 [2:16:19<51:12:15, 10.75s/it]
  5% 855/18000 [2:16:30<51:18:10, 10.77s/it]
  5% 856/18000 [2:16:40<49:38:06, 10.42s/it]
  5% 857/18000 [2:16:48<46:51:14,  9.84s/it]
  5% 858/18000 [2:16:55<42:52:23,  9.00s/it]
  5% 859/18000 [2:17:01<38:36:12,  8.11s/it]
  5% 860/18000 [2:17:06<33:29:36,  7.03s/it]
  5% 861/18000 [2:17:21<45:39:57,  9.59s/it]
  5% 862/18000 [2:17:31<46:13:32,  9.71s/it]
  5% 863/18000 [2:17:38<42:43:36,  8.98s/it]
  5% 864/18000 [2:17:42<34:32:47,  7.26s/it]
  5% 865/18000 [2:17:58<46:51:32,  9.84s/it]
  5% 866/18000 [2:18:11<51:37:38, 10.85s/it]
  5% 867/18000 [2:18:22<51:42:44, 10.87s/it]
  5% 868/18000 [2:18:31<49:19:19, 10.36s/it]
  5% 869/18000 [2:18:39<46:19:01,  9.73s/it]
  5% 870/18000 [2:18:46<42:35:52,  8.95s/it]
  5% 871/18000 [2:18:52<38:07:53,  8.01s/it]
  5% 872/18000 [2:18:56<32:56:43,  6.92s/it]
  5% 873/18000 [2:19:13<46:42:21,  9.82s/it]
  5% 874/18000 [2:19:27<51:57:22, 10.92s/it]
  5% 875/18000 [2:19:37<51:54:02, 10.91s/it]
  5% 876/18000 [2:19:47<49:58:37, 10.51s/it]
  5% 877/18000 [2:19:55<47:03:53,  9.90s/it]
  5% 878/18000 [2:20:03<43:06:02,  9.06s/it]
  5% 879/18000 [2:20:09<38:46:49,  8.15s/it]
  5% 880/18000 [2:20:13<33:31:57,  7.05s/it]
  5% 881/18000 [2:20:30<47:18:11,  9.95s/it]
  5% 882/18000 [2:20:43<52:22:34, 11.01s/it]
  5% 883/18000 [2:20:54<52:06:44, 10.96s/it]
  5% 884/18000 [2:21:04<49:55:09, 10.50s/it]
  5% 885/18000 [2:21:12<46:17:31,  9.74s/it]
  5% 886/18000 [2:21:18<42:02:10,  8.84s/it]
  5% 887/18000 [2:21:24<37:27:36,  7.88s/it]
  5% 888/18000 [2:21:28<32:30:24,  6.84s/it]
  5% 889/18000 [2:21:45<46:27:48,  9.78s/it]
  5% 890/18000 [2:21:58<51:44:59, 10.89s/it]
  5% 891/18000 [2:22:09<51:39:52, 10.87s/it]
  5% 892/18000 [2:22:18<49:15:15, 10.36s/it]
  5% 893/18000 [2:22:26<45:57:44,  9.67s/it]
  5% 894/18000 [2:22:34<42:18:26,  8.90s/it]
  5% 895/18000 [2:22:40<38:13:28,  8.04s/it]
  5% 896/18000 [2:22:44<33:15:11,  7.00s/it]
  5% 897/18000 [2:22:59<44:53:29,  9.45s/it]
  5% 898/18000 [2:23:09<45:41:26,  9.62s/it]
  5% 899/18000 [2:23:17<42:46:37,  9.01s/it]
  5% 900/18000 [2:23:20<34:43:23,  7.31s/it]
                                            
  5% 900/18000 [2:23:20<34:43:23,  7.31s/it]
  5% 901/18000 [2:23:37<48:15:08, 10.16s/it]
  5% 902/18000 [2:23:51<52:57:29, 11.15s/it]
  5% 903/18000 [2:24:01<52:31:41, 11.06s/it]
  5% 904/18000 [2:24:11<50:12:12, 10.57s/it]
  5% 905/18000 [2:24:19<46:52:48,  9.87s/it]
  5% 906/18000 [2:24:26<42:48:42,  9.02s/it]
  5% 907/18000 [2:24:32<38:14:04,  8.05s/it]
  5% 908/18000 [2:24:36<33:05:21,  6.97s/it]
  5% 909/18000 [2:24:53<46:21:38,  9.77s/it]
  5% 910/18000 [2:25:05<50:27:44, 10.63s/it]
  5% 911/18000 [2:25:16<50:14:39, 10.58s/it]
  5% 912/18000 [2:25:25<48:18:39, 10.18s/it]
  5% 913/18000 [2:25:33<45:29:02,  9.58s/it]
  5% 914/18000 [2:25:40<41:43:59,  8.79s/it]
  5% 915/18000 [2:25:46<37:21:46,  7.87s/it]
  5% 916/18000 [2:25:50<32:22:09,  6.82s/it]
  5% 917/18000 [2:26:07<46:02:15,  9.70s/it]
  5% 918/18000 [2:26:20<51:15:25, 10.80s/it]
  5% 919/18000 [2:26:31<50:57:58, 10.74s/it]
  5% 920/18000 [2:26:40<48:55:32, 10.31s/it]
  5% 921/18000 [2:26:48<46:17:15,  9.76s/it]
  5% 922/18000 [2:26:55<42:07:35,  8.88s/it]
  5% 923/18000 [2:27:01<37:48:05,  7.97s/it]
  5% 924/18000 [2:27:06<32:53:21,  6.93s/it]
  5% 925/18000 [2:27:22<46:45:08,  9.86s/it]
  5% 926/18000 [2:27:35<51:25:12, 10.84s/it]
  5% 927/18000 [2:27:46<51:37:03, 10.88s/it]
  5% 928/18000 [2:27:56<49:50:45, 10.51s/it]
  5% 929/18000 [2:28:04<46:45:26,  9.86s/it]
  5% 930/18000 [2:28:12<42:50:33,  9.04s/it]
  5% 931/18000 [2:28:17<38:20:45,  8.09s/it]
  5% 932/18000 [2:28:22<33:04:13,  6.98s/it]
  5% 933/18000 [2:28:37<45:08:13,  9.52s/it]
  5% 934/18000 [2:28:47<46:07:58,  9.73s/it]
  5% 935/18000 [2:28:55<42:41:15,  9.01s/it]
  5% 936/18000 [2:28:58<34:48:45,  7.34s/it]
  5% 937/18000 [2:29:15<47:56:41, 10.12s/it]
  5% 938/18000 [2:29:28<52:50:25, 11.15s/it]
  5% 939/18000 [2:29:39<52:06:36, 11.00s/it]
  5% 940/18000 [2:29:49<50:06:02, 10.57s/it]
  5% 941/18000 [2:29:57<46:27:46,  9.81s/it]
  5% 942/18000 [2:30:04<42:28:04,  8.96s/it]
  5% 943/18000 [2:30:09<38:04:14,  8.04s/it]
  5% 944/18000 [2:30:14<32:53:41,  6.94s/it]
  5% 945/18000 [2:30:31<46:41:06,  9.85s/it]
  5% 946/18000 [2:30:44<51:45:01, 10.92s/it]
  5% 947/18000 [2:30:55<51:52:08, 10.95s/it]
  5% 948/18000 [2:31:05<50:01:31, 10.56s/it]
  5% 949/18000 [2:31:13<47:04:00,  9.94s/it]
  5% 950/18000 [2:31:20<43:22:58,  9.16s/it]
  5% 951/18000 [2:31:27<39:03:02,  8.25s/it]
  5% 952/18000 [2:31:31<33:41:26,  7.11s/it]
  5% 953/18000 [2:31:47<46:47:28,  9.88s/it]
  5% 954/18000 [2:32:00<50:51:54, 10.74s/it]
  5% 955/18000 [2:32:11<50:29:31, 10.66s/it]
  5% 956/18000 [2:32:20<48:30:28, 10.25s/it]
  5% 957/18000 [2:32:28<45:36:33,  9.63s/it]
  5% 958/18000 [2:32:35<41:48:30,  8.83s/it]
  5% 959/18000 [2:32:41<37:14:18,  7.87s/it]
  5% 960/18000 [2:32:45<32:30:47,  6.87s/it]
  5% 961/18000 [2:33:02<46:12:50,  9.76s/it]
  5% 962/18000 [2:33:15<51:02:04, 10.78s/it]
  5% 963/18000 [2:33:26<50:56:33, 10.76s/it]
  5% 964/18000 [2:33:35<49:06:52, 10.38s/it]
  5% 965/18000 [2:33:43<46:06:46,  9.75s/it]
  5% 966/18000 [2:33:50<42:09:31,  8.91s/it]
  5% 967/18000 [2:33:56<37:39:04,  7.96s/it]
  5% 968/18000 [2:34:00<32:37:48,  6.90s/it]
  5% 969/18000 [2:34:16<45:21:57,  9.59s/it]
  5% 970/18000 [2:34:27<46:34:26,  9.85s/it]
  5% 971/18000 [2:34:34<43:15:07,  9.14s/it]
  5% 972/18000 [2:34:38<35:11:45,  7.44s/it]
  5% 973/18000 [2:34:54<48:18:52, 10.22s/it]
  5% 974/18000 [2:35:08<52:51:04, 11.17s/it]
  5% 975/18000 [2:35:19<52:22:27, 11.07s/it]
  5% 976/18000 [2:35:28<50:10:37, 10.61s/it]
  5% 977/18000 [2:35:37<47:06:13,  9.96s/it]
  5% 978/18000 [2:35:44<42:54:50,  9.08s/it]
  5% 979/18000 [2:35:50<38:20:31,  8.11s/it]
  5% 980/18000 [2:35:54<33:10:36,  7.02s/it]
  5% 981/18000 [2:36:11<46:47:34,  9.90s/it]
  5% 982/18000 [2:36:24<51:53:27, 10.98s/it]
  5% 983/18000 [2:36:35<51:53:32, 10.98s/it]
  5% 984/18000 [2:36:45<49:50:51, 10.55s/it]
  5% 985/18000 [2:36:53<46:36:48,  9.86s/it]
  5% 986/18000 [2:37:00<42:32:11,  9.00s/it]
  5% 987/18000 [2:37:05<37:44:50,  7.99s/it]
  5% 988/18000 [2:37:10<32:36:42,  6.90s/it]
  5% 989/18000 [2:37:27<46:29:35,  9.84s/it]
  6% 990/18000 [2:37:40<51:56:01, 10.99s/it]
  6% 991/18000 [2:37:51<51:45:07, 10.95s/it]
  6% 992/18000 [2:38:01<49:51:26, 10.55s/it]
  6% 993/18000 [2:38:09<46:53:43,  9.93s/it]
  6% 994/18000 [2:38:16<42:59:08,  9.10s/it]
  6% 995/18000 [2:38:22<38:38:42,  8.18s/it]
  6% 996/18000 [2:38:27<33:25:18,  7.08s/it]
  6% 997/18000 [2:38:43<45:48:34,  9.70s/it]
  6% 998/18000 [2:38:56<50:37:29, 10.72s/it]
  6% 999/18000 [2:39:06<50:27:14, 10.68s/it]
  6% 1000/18000 [2:39:16<48:50:10, 10.34s/it]
                                             
  6% 1000/18000 [2:39:16<48:50:10, 10.34s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+
+
  0% 0/33 [00:00<?, ?it/s]
+
  6% 2/33 [00:04<01:10,  2.27s/it]
+
  9% 3/33 [00:09<01:35,  3.19s/it]
+
 12% 4/33 [00:13<01:41,  3.52s/it]
+
 15% 5/33 [00:17<01:44,  3.72s/it]
+
 18% 6/33 [00:21<01:44,  3.88s/it]
+
 21% 7/33 [00:25<01:40,  3.87s/it]
+
 24% 8/33 [00:29<01:38,  3.92s/it]
+
 27% 9/33 [00:33<01:34,  3.95s/it]
+
 30% 10/33 [00:37<01:35,  4.14s/it]
+
 33% 11/33 [00:41<01:30,  4.12s/it]
+
 36% 12/33 [00:45<01:25,  4.09s/it]
+
 39% 13/33 [00:49<01:20,  4.03s/it]
+
 42% 14/33 [00:54<01:17,  4.09s/it]
+
 45% 15/33 [00:58<01:13,  4.11s/it]
+
 48% 16/33 [01:02<01:12,  4.24s/it]
+
 52% 17/33 [01:07<01:10,  4.38s/it]
+
 55% 18/33 [01:11<01:04,  4.31s/it]
+
 58% 19/33 [01:15<01:00,  4.31s/it]
+
 61% 20/33 [01:20<00:57,  4.39s/it]
+
 64% 21/33 [01:25<00:53,  4.43s/it]
+
 67% 22/33 [01:29<00:48,  4.41s/it]
+
 70% 23/33 [01:34<00:44,  4.46s/it]
+
 73% 24/33 [01:38<00:40,  4.47s/it]
+
 76% 25/33 [01:43<00:36,  4.52s/it]
+
 79% 26/33 [01:47<00:31,  4.47s/it]
+
 82% 27/33 [01:51<00:26,  4.43s/it]
+
 85% 28/33 [01:56<00:22,  4.46s/it]
+
 88% 29/33 [01:59<00:16,  4.20s/it]
+
 91% 30/33 [02:02<00:11,  3.80s/it]
+
 94% 31/33 [02:05<00:07,  3.50s/it]
+
 97% 32/33 [02:08<00:03,  3.25s/it]
+
                                             
+
                                   
  6% 1000/18000 [2:41:32<48:50:10, 10.34s/it]
+
+
                                   Saving model checkpoint to ./checkpoint-1000
+Configuration saved in ./checkpoint-1000/config.json
+Model weights saved in ./checkpoint-1000/pytorch_model.bin
+Configuration saved in ./checkpoint-1000/preprocessor_config.json
 Configuration saved in ./preprocessor_config.json

logs

@@ -97,3 +97,9 @@ xpu_backend=None,
 {'loss': 1.2823, 'learning_rate': 0.000598, 'epoch': 8.33}
 {'loss': 0.3962, 'learning_rate': 0.0007980000000000001, 'epoch': 11.11}
 {'loss': 0.3034, 'learning_rate': 0.000998, 'epoch': 13.89}
+{'eval_loss': 0.38567885756492615, 'eval_wer': 0.4354438434616608, 'eval_runtime': 133.1769, 'eval_samples_per_second': 19.59, 'eval_steps_per_second': 0.248, 'epoch': 13.89}
+{'loss': 0.2639, 'learning_rate': 0.0009999210371864342, 'epoch': 16.67}
+{'loss': 0.2354, 'learning_rate': 0.0009996809758056298, 'epoch': 19.44}
+{'loss': 0.2158, 'learning_rate': 0.000999279885163942, 'epoch': 22.22}
+{'loss': 0.2063, 'learning_rate': 0.000998717894518502, 'epoch': 25.0}
+{'loss': 0.1926, 'learning_rate': 0.000997995184978743, 'epoch': 27.78}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3aae5e50ef4a8b89fd5fb88e1636f3e95144e39211b78b0a3f3767d443d62958
+oid sha256:9522e4f73638d2cb960a7316bc2943a69e7b1f68581d9639de3e1b9b05dc8530
 size 1262083569

wandb/run-20220126_105847-1xdiy2kf/files/output.log

@@ -542,3 +542,547 @@
 Configuration saved in ./checkpoint-500/config.json
 Model weights saved in ./checkpoint-500/pytorch_model.bin
 Configuration saved in ./checkpoint-500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  6% 1000/18000 [2:39:16<48:50:10, 10.34s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+  0% 0/33 [00:00<?, ?it/s]
+{'eval_loss': 0.38567885756492615, 'eval_wer': 0.4354438434616608, 'eval_runtime': 133.1769, 'eval_samples_per_second': 19.59, 'eval_steps_per_second': 0.248, 'epoch': 13.89}
+{'loss': 0.2639, 'learning_rate': 0.0009999210371864342, 'epoch': 16.67}
+{'loss': 0.2354, 'learning_rate': 0.0009996809758056298, 'epoch': 19.44}
+{'loss': 0.2158, 'learning_rate': 0.000999279885163942, 'epoch': 22.22}
+{'loss': 0.2063, 'learning_rate': 0.000998717894518502, 'epoch': 25.0}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Configuration saved in ./checkpoint-1000/config.json
+Model weights saved in ./checkpoint-1000/pytorch_model.bin
+Configuration saved in ./checkpoint-1000/preprocessor_config.json

wandb/run-20220126_121554-295enqix/files/output.log

@@ -10,6 +10,18 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

wandb/run-20220126_121554-295enqix/logs/debug-internal.log

@@ -85,3 +85,52 @@
 2022-01-26 12:19:23,373 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
 2022-01-26 12:19:28,370 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
 2022-01-26 12:19:28,371 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:19:30,192 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:19:33,379 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:19:41,382 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:19:43,543 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:19:43,544 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:19:51,386 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:19:58,761 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:19:58,761 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:19:59,389 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:01,184 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:20:07,393 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:13,395 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:13,950 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:20:13,950 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:20:19,398 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:25,401 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:29,257 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:20:29,257 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:20:29,402 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:32,140 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:20:44,546 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:20:44,547 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:20:47,411 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:20:59,689 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:20:59,690 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:21:03,070 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
+2022-01-26 12:21:04,418 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:21:14,840 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
+2022-01-26 12:21:14,840 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
+2022-01-26 12:21:18,425 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:21:32,200 WARNING MainThread:2226380 [internal.py:is_dead():366] Internal process exiting, parent pid 2226158 disappeared
+2022-01-26 12:21:32,200 ERROR   MainThread:2226380 [internal.py:wandb_internal():142] Internal process shutdown.
+2022-01-26 12:21:32,483 INFO    SenderThread:2226380 [sender.py:finish():1070] shutting down sender
+2022-01-26 12:21:32,484 INFO    SenderThread:2226380 [dir_watcher.py:finish():283] shutting down directory watcher
+2022-01-26 12:21:32,492 INFO    HandlerThread:2226380 [handler.py:finish():731] shutting down handler
+2022-01-26 12:21:32,503 INFO    WriterThread:2226380 [datastore.py:close():281] close: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/run-295enqix.wandb
+2022-01-26 12:21:33,428 INFO    SenderThread:2226380 [dir_watcher.py:finish():313] scan: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files
+2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/config.yaml config.yaml
+2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-summary.json wandb-summary.json
+2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/requirements.txt requirements.txt
+2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-metadata.json wandb-metadata.json
+2022-01-26 12:21:33,430 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log output.log
+2022-01-26 12:21:33,430 INFO    SenderThread:2226380 [file_pusher.py:finish():177] shutting down file pusher
+2022-01-26 12:21:33,430 INFO    SenderThread:2226380 [file_pusher.py:join():182] waiting for file pusher
+2022-01-26 12:21:34,064 INFO    Thread-14 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/requirements.txt
+2022-01-26 12:21:34,092 INFO    Thread-13 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-summary.json
+2022-01-26 12:21:34,134 INFO    Thread-12 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/config.yaml
+2022-01-26 12:21:34,181 INFO    Thread-15 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
+2022-01-26 12:21:34,621 INFO    MainThread:2226380 [internal.py:handle_exit():77] Internal process exited