Training in progress, step 1000

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:959730eeb43c0cd843bb45b7a4767db1272b1b23273ec245695c7c947d734279
+oid sha256:a522c62e673b9c8d1eb07e9cb3438aae2df6cc0078ede1ce8eb1476cfb12f289
 size 1262075377

runs/Jan27_12-08-15_brutasse/events.out.tfevents.1643285566.brutasse.15970.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:faa1c9454d06b46404123fa8ca214be5ae73cabac14a7b8d7a36a4ea3965c965
-size 5781
+oid sha256:499f1e961bff4742ca8723a8738bc46df086899c0a1623a5551cbb94c04b3683
+size 6884

wandb/run-20220127_121247-o8za41vn/files/config.yaml

@@ -4782,6 +4782,26 @@ _wandb:
       5: 1
       6:
       - 1
+    - 1: eval/loss
+      5: 1
+      6:
+      - 1
+    - 1: eval/wer
+      5: 1
+      6:
+      - 1
+    - 1: eval/runtime
+      5: 1
+      6:
+      - 1
+    - 1: eval/samples_per_second
+      5: 1
+      6:
+      - 1
+    - 1: eval/steps_per_second
+      5: 1
+      6:
+      - 1
     python_version: 3.8.12
     start_time: 1643285567
     t:

wandb/run-20220127_121247-o8za41vn/files/output.log

@@ -382,3 +382,384 @@
 Configuration saved in ./checkpoint-500/config.json
 Model weights saved in ./checkpoint-500/pytorch_model.bin
 Configuration saved in ./checkpoint-500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|████▊                                                                                                                                              | 600/18250 [14:49<5:22:19,  1.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|█████▋                                                                                                                                             | 701/18250 [16:51<5:15:43,  1.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|██████▍                                                                                                                                            | 799/18250 [18:51<6:36:30,  1.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|███████▏                                                                                                                                           | 900/18250 [20:52<6:21:51,  1.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|████████                                                                                                                                          | 1000/18250 [22:53<6:24:02,  1.34s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 4843
+  Batch size = 8
+  0%|▌                                                                                                                                                        | 2/606 [00:00<01:52,  5.38it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 605/606 [02:22<00:00,  4.04it/s]
+Configuration saved in ./checkpoint-1000/config.json
+Model weights saved in ./checkpoint-1000/pytorch_model.bin
+Configuration saved in ./checkpoint-1000/preprocessor_config.json

wandb/run-20220127_121247-o8za41vn/logs/debug-internal.log

@@ -1758,3 +1758,537 @@
 2022-01-27 12:25:31,498 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
 2022-01-27 12:25:31,498 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
 2022-01-27 12:25:32,223 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:44,226 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:46,226 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:46,593 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:25:46,593 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:25:48,227 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:50,227 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:52,227 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:54,004 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:25:54,228 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:57,228 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/config.yaml
+2022-01-27 12:25:57,228 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:25:59,229 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:01,229 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:01,714 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:26:01,714 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:26:03,229 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:05,230 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:07,230 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:09,230 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:11,231 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:13,231 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:15,231 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:16,804 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:26:16,804 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:26:17,232 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:19,232 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:21,232 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:23,233 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:24,339 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:26:25,233 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:27,233 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:29,234 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:31,234 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:31,877 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:26:31,877 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:26:33,234 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:35,235 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:37,235 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:39,235 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:41,235 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:43,236 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:45,236 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:47,019 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:26:47,019 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:26:47,236 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:49,237 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:51,237 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:53,237 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:54,606 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:26:55,238 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:57,238 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:26:59,238 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:01,239 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:02,093 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:27:02,093 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:27:03,239 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:05,239 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:07,239 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:09,240 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:11,240 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:13,240 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:15,241 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:17,169 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:27:17,169 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:27:17,241 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:19,242 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:21,242 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:23,242 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:24,867 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:27:25,242 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:27,243 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:29,243 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:31,243 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:32,279 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:27:32,279 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:27:33,244 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:35,244 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:37,244 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:40,245 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:42,245 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:43,013 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:27:43,058 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:27:43,082 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:27:43,245 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:27:44,245 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:45,246 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:46,246 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:47,371 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:27:47,371 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:27:48,246 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:50,246 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:52,247 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:54,247 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:55,215 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:27:56,247 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:27:58,248 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:00,248 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:02,249 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:02,467 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:28:02,467 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:28:04,249 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:06,249 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:08,250 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:10,250 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:12,250 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:14,251 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:16,251 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:17,533 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:28:17,533 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:28:18,251 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:20,252 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:22,252 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:24,252 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:25,478 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:28:26,253 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:28,253 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:30,253 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:32,253 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:32,668 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:28:32,669 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:28:34,254 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:36,254 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:38,254 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:40,255 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:42,255 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:44,255 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:46,255 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:47,736 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:28:47,736 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:28:48,256 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:50,256 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:52,256 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:54,257 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:55,746 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:28:56,257 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:28:58,257 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:00,258 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:02,258 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:02,870 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:29:02,870 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:29:04,258 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:06,258 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:08,259 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:10,259 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:12,259 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:14,260 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:16,260 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:17,995 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:29:17,995 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:29:19,261 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:21,261 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:23,261 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:25,262 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:25,991 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:29:27,262 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:29,262 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:31,263 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:33,137 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:29:33,137 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:29:33,263 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:35,263 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:37,263 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:39,264 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:41,264 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:43,196 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:29:43,247 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:29:43,270 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:43,271 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:29:44,270 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:29:45,270 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:47,271 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:48,215 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:29:48,216 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:29:49,271 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:51,271 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:53,272 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:55,272 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:56,260 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:29:57,272 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:29:59,273 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:01,273 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:03,273 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:03,312 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:30:03,312 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:30:05,274 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:07,274 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:09,274 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:11,275 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:13,275 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:15,275 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:17,276 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:18,459 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:30:18,459 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:30:19,276 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:21,276 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:23,277 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:25,277 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:26,520 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:30:27,277 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:29,278 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:31,278 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:33,278 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:33,536 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:30:33,536 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:30:35,279 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:37,279 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:39,279 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:41,280 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:43,280 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:45,280 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:47,281 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:48,632 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:30:48,632 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:30:49,281 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:51,281 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:53,281 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:55,282 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:56,775 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:30:57,282 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:30:59,282 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:01,283 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:03,283 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:03,713 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:31:03,713 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:31:05,283 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:07,284 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:09,284 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:11,284 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:13,285 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:15,285 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:17,285 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:18,784 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:31:18,784 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:31:19,286 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:21,286 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:23,286 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:25,286 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:27,021 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:31:27,287 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:29,287 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:31,287 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:33,288 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:33,911 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:31:33,911 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:31:35,288 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:37,288 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:40,289 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:42,289 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:44,289 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:45,461 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:31:45,504 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:31:45,530 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:31:46,290 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:31:46,290 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:47,290 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:48,290 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:48,982 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:31:48,983 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:31:50,290 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:52,291 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:54,291 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:56,291 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:31:57,277 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:31:58,292 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:00,292 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:02,292 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:04,072 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:32:04,072 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:32:04,293 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:06,293 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:08,293 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:10,294 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:12,294 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:14,294 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:16,295 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:18,295 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:19,207 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:32:19,207 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:32:20,295 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:22,296 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:24,296 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:26,296 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:27,536 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:32:28,297 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:30,297 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:32,297 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:34,298 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:34,322 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:32:34,322 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:32:36,298 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:38,298 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:40,298 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:42,299 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:44,299 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:46,299 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:48,300 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:49,424 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:32:49,424 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:32:50,300 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:52,300 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:54,301 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:56,301 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:32:57,782 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:32:58,301 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:00,302 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:02,302 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:04,302 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:04,494 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:33:04,494 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:33:06,303 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:08,303 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:10,303 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:12,304 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:14,304 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:16,304 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:18,304 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:19,566 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:33:19,566 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:33:20,305 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:22,305 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:24,305 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:26,306 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:28,049 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:33:28,306 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:30,306 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:32,307 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:34,307 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:34,824 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:33:34,824 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:33:36,307 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:38,308 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:40,308 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:42,308 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:44,308 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:45,894 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:33:45,939 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:33:45,963 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:33:46,309 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:33:46,309 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:48,309 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:49,948 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:33:49,948 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:33:50,309 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:52,310 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:55,310 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:57,310 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:33:58,296 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:33:59,311 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:01,311 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:03,311 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:05,056 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:34:05,057 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:34:05,312 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:07,312 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:09,312 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:11,313 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:13,313 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:15,313 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:17,314 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:20,138 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:34:20,138 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:34:21,314 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:23,315 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:25,315 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:27,315 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:28,549 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:34:29,316 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:31,316 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:33,316 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:35,214 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:34:35,214 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:34:35,316 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:37,317 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:39,317 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:41,317 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:43,318 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:45,318 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:47,318 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:49,319 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:50,303 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:34:50,303 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:34:51,319 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:53,319 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:55,320 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:57,320 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:34:58,803 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:34:59,320 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:01,321 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:03,321 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:05,321 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:05,400 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:35:05,400 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:35:07,322 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:09,322 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:11,322 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:13,323 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:15,323 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:17,323 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:19,324 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:20,516 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:35:20,516 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:35:21,324 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:23,324 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:25,324 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:27,325 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:29,059 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:35:29,325 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:31,325 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:33,326 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:35,326 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:35,592 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:35:35,593 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:35:37,326 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:39,327 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:41,327 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:43,327 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:45,328 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:46,206 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:35:46,252 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:35:46,276 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:35:46,328 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:35:47,328 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:48,328 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:49,328 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:50,672 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:35:50,672 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:35:51,329 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:54,329 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:56,329 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:58,330 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:35:59,324 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:36:00,330 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:02,330 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:04,331 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:05,779 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:36:05,779 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:36:06,331 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:08,332 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:10,332 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:12,332 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:14,332 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:16,333 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:18,333 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:20,333 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:20,891 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:36:20,891 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:36:22,334 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:24,334 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:26,334 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:28,335 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:29,588 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:36:30,335 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:32,335 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:34,336 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:36,112 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:36:36,112 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:36:36,336 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:38,336 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:40,337 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:42,337 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:44,337 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:46,338 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:48,338 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:50,338 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:51,215 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:36:51,215 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:36:52,339 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:54,339 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:56,339 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:58,340 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:36:59,882 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:37:00,340 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:02,340 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:04,340 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:06,302 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:37:06,302 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:37:06,341 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:08,341 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:11,342 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:13,342 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:15,342 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:17,343 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:19,343 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:21,343 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:21,372 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:37:21,372 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:37:23,344 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:25,344 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:27,345 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:29,345 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:30,160 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:37:31,345 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:33,345 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:35,346 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:36,508 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:37:36,508 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:37:37,346 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:39,347 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:41,347 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:43,347 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:45,348 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:47,348 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:49,348 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:51,349 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:51,587 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:37:51,588 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:37:53,349 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:55,349 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:57,350 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:37:59,350 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:00,423 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:38:01,350 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:03,351 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:05,351 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:06,752 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:38:06,752 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:38:07,351 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:09,352 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:11,051 DEBUG   SenderThread:16113 [sender.py:send():182] send: history
+2022-01-27 12:38:11,069 DEBUG   SenderThread:16113 [sender.py:send():182] send: summary
+2022-01-27 12:38:11,099 INFO    SenderThread:16113 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-27 12:38:11,352 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/wandb-summary.json
+2022-01-27 12:38:13,353 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:21,828 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:38:21,828 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:38:30,725 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:38:36,902 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:38:36,902 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:38:43,363 INFO    Thread-8  :16113 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/experiments/xls-r-300m-sv-cv8/wandb/run-20220127_121247-o8za41vn/files/output.log
+2022-01-27 12:38:51,979 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:38:51,979 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status
+2022-01-27 12:39:01,031 DEBUG   SenderThread:16113 [sender.py:send():182] send: stats
+2022-01-27 12:39:07,057 DEBUG   HandlerThread:16113 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-27 12:39:07,057 DEBUG   SenderThread:16113 [sender.py:send_request():196] send_request: stop_status