Training in progress, step 1500

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a522c62e673b9c8d1eb07e9cb3438aae2df6cc0078ede1ce8eb1476cfb12f289
+oid sha256:4d80289eb039576f75e350675c60903ba8fcc6cc2d64287efc8fe26ccf4abf09
 size 1262075377

runs/Jan27_12-08-15_brutasse/events.out.tfevents.1643285566.brutasse.15970.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:499f1e961bff4742ca8723a8738bc46df086899c0a1623a5551cbb94c04b3683
-size 6884
+oid sha256:a2ea9ccd8501b33656ca7e02d6de78e045a6244eede76ffeff590957e662a184
+size 7987

wandb/run-20220127_121247-o8za41vn/files/output.log

@@ -763,3 +763,383 @@ Configuration saved in ./preprocessor_config.json
 Configuration saved in ./checkpoint-1000/config.json
 Model weights saved in ./checkpoint-1000/pytorch_model.bin
 Configuration saved in ./checkpoint-1000/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  6%|████████▊                                                                                                                                         | 1099/18250 [28:17<8:13:33,  1.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████▌                                                                                                                                        | 1200/18250 [30:20<7:44:56,  1.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|██████████▍                                                                                                                                       | 1300/18250 [32:21<7:45:42,  1.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████▏                                                                                                                                      | 1399/18250 [34:20<7:37:05,  1.63s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|████████████                                                                                                                                      | 1500/18250 [36:24<4:36:19,  1.01it/s]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 4843
+  Batch size = 8
+{'loss': 2.1173, 'learning_rate': 5.62125e-05, 'epoch': 4.11}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Configuration saved in ./checkpoint-1500/config.json
+Model weights saved in ./checkpoint-1500/pytorch_model.bin
+Configuration saved in ./checkpoint-1500/preprocessor_config.json
+{'eval_loss': 1.147773265838623, 'eval_wer': 0.8787680953732614, 'eval_runtime': 146.3922, 'eval_samples_per_second': 33.082, 'eval_steps_per_second': 4.14, 'epoch': 4.11}

wandb/run-20220127_121247-o8za41vn/logs/debug-internal.log

@@ -2292,3 +2292,537 @@
 2022-01-27 12:39:01,031 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
 2022-01-27 12:39:07,057 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
 2022-01-27 12:39:07,057 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:39:17,371 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:19,371 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:22,139 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:39:22,140 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:39:22,372 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:24,372 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:26,373 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:28,373 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:30,373 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:31,308 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:39:32,374 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:34,374 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:36,375 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:37,241 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:39:37,241 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:39:38,375 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:40,375 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:42,376 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:44,376 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:46,376 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:48,377 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:50,377 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:52,377 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:52,391 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:39:52,391 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:39:54,378 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:56,378 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:39:58,378 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:00,379 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:01,559 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:40:02,379 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:04,379 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:06,380 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:07,502 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:40:07,502 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:40:08,380 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:10,380 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:12,380 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:14,381 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:16,381 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:18,382 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:20,382 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:22,383 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:22,591 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:40:22,591 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:40:24,383 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:26,383 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:28,384 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:30,384 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:31,829 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:40:32,384 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:34,385 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:36,385 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:37,714 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:40:37,715 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:40:38,385 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:40,386 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:42,386 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:44,386 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:46,387 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:48,387 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:50,387 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:52,388 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:52,819 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:40:52,819 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:40:54,388 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:56,388 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:40:58,389 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:00,389 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:02,088 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:41:02,389 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:04,390 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:06,390 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:08,029 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:41:08,029 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:41:08,391 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:10,391 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:12,392 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:12,763 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:41:12,817 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:41:12,841 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:41:13,392 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:41:13,392 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:14,392 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:16,393 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:18,393 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:20,394 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:22,394 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:23,139 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:41:23,140 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:41:24,395 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:26,395 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:28,396 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:30,396 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:32,351 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:41:32,397 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:34,397 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:36,398 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:38,247 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:41:38,247 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:41:38,398 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:40,399 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:42,399 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:44,400 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:46,400 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:48,401 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:50,401 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:52,401 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:53,345 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:41:53,346 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:41:54,402 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:56,402 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:41:58,403 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:00,403 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:02,404 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:02,606 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:42:04,404 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:06,405 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:08,482 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:42:08,482 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:42:09,405 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:11,406 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:13,406 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:15,407 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:17,407 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:19,408 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:21,408 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:23,409 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:23,560 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:42:23,560 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:42:25,409 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:27,410 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:29,410 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:31,411 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:32,875 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:42:33,411 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:35,412 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:37,412 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:39,202 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:42:39,202 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:42:39,413 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:41,413 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:43,413 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:45,414 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:47,414 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:49,415 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:51,415 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:53,416 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:54,293 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:42:54,293 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:42:55,416 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:57,417 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:42:59,417 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:01,418 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:03,143 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:43:03,418 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:05,419 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:07,419 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:09,412 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:43:09,412 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:43:09,420 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:11,420 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:13,421 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:13,716 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:43:13,761 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:43:13,792 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:43:14,421 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:43:15,421 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:16,421 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:17,422 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:19,422 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:21,423 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:23,423 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:24,492 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:43:24,492 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:43:25,424 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:27,424 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:29,424 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:31,425 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:33,409 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:43:33,425 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:35,426 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:37,426 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:39,427 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:39,567 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:43:39,567 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:43:41,427 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:43,428 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:45,428 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:47,429 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:49,429 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:51,430 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:53,430 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:54,666 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:43:54,667 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:43:55,431 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:57,431 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:43:59,432 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:01,432 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:03,433 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:03,666 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:44:05,433 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:09,434 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:09,807 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:44:09,807 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:44:11,435 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:13,435 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:15,436 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:17,436 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:19,436 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:21,437 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:23,437 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:24,945 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:44:24,946 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:44:25,438 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:27,438 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:29,439 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:31,439 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:33,440 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:33,929 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:44:35,440 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:37,441 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:40,066 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:44:40,066 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:44:40,441 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:42,442 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:44,442 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:46,443 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:48,443 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:50,444 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:52,444 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:54,445 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:55,418 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:44:55,418 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:44:56,445 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:44:58,446 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:00,446 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:02,447 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:04,194 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:45:04,447 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:06,448 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:08,448 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:10,449 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:10,500 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:45:10,500 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:45:12,449 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:14,247 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:45:14,286 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:45:14,310 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:45:14,450 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:45:14,450 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:16,450 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:18,450 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:20,451 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:22,452 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:24,452 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:25,581 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:45:25,582 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:45:26,453 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:28,453 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:30,453 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:32,454 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:34,454 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:34,500 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:45:36,455 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:38,456 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:40,456 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:40,662 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:45:40,662 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:45:42,457 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:44,457 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:46,457 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:48,458 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:50,458 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:52,459 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:54,459 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:55,792 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:45:55,792 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:45:56,460 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:45:58,460 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:00,461 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:02,461 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:04,462 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:04,748 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:46:06,462 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:08,463 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:10,463 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:10,912 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:46:10,912 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:46:12,464 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:14,464 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:16,465 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:18,465 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:20,466 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:22,466 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:24,466 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:26,097 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:46:26,097 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:46:26,467 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:28,467 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:30,468 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:32,468 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:34,469 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:34,994 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:46:36,469 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:38,470 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:40,470 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:41,175 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:46:41,175 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:46:42,470 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:45,471 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:47,472 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:49,472 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:51,473 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:53,473 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:55,474 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:56,279 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:46:56,279 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:46:57,474 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:46:59,474 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:01,475 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:03,475 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:05,238 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:47:05,476 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:07,476 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:09,477 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:11,355 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:47:11,355 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:47:11,478 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:13,478 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:15,255 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:47:15,306 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:47:15,329 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:47:15,478 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:47:15,479 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:17,479 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:19,479 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:21,480 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:23,480 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:25,481 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:26,475 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:47:26,475 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:47:27,481 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:29,482 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:31,482 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:33,483 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:35,483 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:35,508 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:47:37,484 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:39,484 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:41,485 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:41,602 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:47:41,602 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:47:43,485 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:45,486 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:47,486 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:49,486 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:51,487 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:53,487 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:55,488 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:56,686 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:47:56,686 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:47:57,488 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:47:59,489 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:01,489 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:03,490 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:05,490 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:05,758 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:48:07,491 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:09,491 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:11,492 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:11,889 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:48:11,889 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:48:13,492 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:15,493 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:17,493 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:19,494 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:21,494 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:23,495 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:25,495 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:26,961 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:48:26,961 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:48:27,496 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:31,497 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:33,497 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:35,497 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:36,000 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:48:37,498 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:39,498 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:41,499 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:42,048 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:48:42,049 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:48:43,499 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:45,500 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:47,500 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:49,500 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:52,501 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:54,502 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:56,502 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:48:57,181 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:48:57,181 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:48:58,503 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:00,503 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:02,503 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:04,504 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:06,250 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:49:06,504 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:08,505 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:10,505 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:12,255 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:49:12,255 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:49:12,506 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:14,506 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:16,507 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:17,294 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:49:17,336 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:49:17,358 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:49:17,507 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:49:18,507 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:19,508 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:20,508 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:22,508 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:24,509 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:26,509 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:27,399 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:49:27,400 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:49:28,510 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:30,510 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:32,511 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:34,511 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:36,512 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:36,517 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:49:38,512 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:40,513 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:42,513 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:42,552 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:49:42,552 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:49:44,514 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:46,514 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:48,515 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:50,515 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:52,516 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:54,516 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:56,516 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:49:57,688 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:49:57,689 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:49:58,517 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:00,517 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:02,518 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:04,518 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:06,519 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:06,768 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:50:08,519 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:10,520 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:12,520 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:12,936 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:50:12,936 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:50:14,521 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:17,521 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:19,522 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:21,522 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:23,523 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:25,523 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:27,524 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:28,102 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:50:28,102 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:50:29,524 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:31,525 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:33,526 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:35,526 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:37,027 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:50:37,527 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:39,527 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:41,528 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:43,278 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:50:43,278 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:50:43,528 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:45,529 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:47,529 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:49,530 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:51,530 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:53,531 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:55,531 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:57,532 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:50:58,395 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:50:58,395 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:50:59,532 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:01,533 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:03,533 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:05,534 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:07,289 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:51:07,534 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:09,535 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:11,535 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:13,486 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:51:13,486 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:51:13,536 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:15,536 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:17,537 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:19,537 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:21,538 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:23,538 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:25,539 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:27,539 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:28,587 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:51:28,587 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:51:29,540 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:31,540 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:33,540 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:35,541 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:37,542 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:37,546 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:51:39,542 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:41,542 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:43,543 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:43,674 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:51:43,689 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:51:43,718 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:51:43,719 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:51:43,719 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:51:44,543 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:51:45,544 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:46,544 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:51:58,794 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:51:58,795 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:52:07,861 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:52:13,874 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:52:13,874 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:52:19,554 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:52:28,950 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:52:28,951 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:52:38,173 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:52:44,038 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:52:44,038 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status