Training in progress, step 1000

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5dceef9ee6d05b9be59f9aeb3b8cf01f3517ad8d22eb7667f403c4d44ad168d4
+oid sha256:4853d0216f7fa258d87bc7419ec7d31398e4e620f56a3d32e767690755ce7af3
 size 1262075377

runs/Jan09_22-00-50_brutasse/events.out.tfevents.1641765760.brutasse.31164.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4bdf16e32eaaee748205b2763ed49851a3a1d5e15252d6afbe03f6b0d632a85
-size 5758
+oid sha256:5452cd998f77c6f5a9102c308056d3e6e70fe3e86852ac3d3d881e546aabab06
+size 6861

wandb/run-20220109_220240-1g372i3v/files/config.yaml

@@ -4782,6 +4782,26 @@ _wandb:
       5: 1
       6:
       - 1
+    - 1: eval/loss
+      5: 1
+      6:
+      - 1
+    - 1: eval/wer
+      5: 1
+      6:
+      - 1
+    - 1: eval/runtime
+      5: 1
+      6:
+      - 1
+    - 1: eval/samples_per_second
+      5: 1
+      6:
+      - 1
+    - 1: eval/steps_per_second
+      5: 1
+      6:
+      - 1
     python_version: 3.8.12
     start_time: 1641765761
     t:

wandb/run-20220109_220240-1g372i3v/files/output.log

@@ -386,3 +386,392 @@ Configuration saved in ./checkpoint-500/config.json
 {'eval_loss': 3.2857718467712402, 'eval_wer': 1.0, 'eval_runtime': 141.2158, 'eval_samples_per_second': 32.716, 'eval_steps_per_second': 4.093, 'epoch': 1.45}
 Model weights saved in ./checkpoint-500/pytorch_model.bin
 Configuration saved in ./checkpoint-500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+/home/patrick/python_bin/transformers/models/wav2vec2/modeling_wav2vec2.py:1107: UserWarning: __floordiv__ is deprecated, and its behavior will change in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values. To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor').
+  return (input_length - kernel_size) // stride + 1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|█▊                                                  | 601/17200 [15:00<6:03:42,  1.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  4%|██                                                  | 698/17200 [17:01<5:09:46,  1.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|██▍                                                 | 799/17200 [19:05<4:56:26,  1.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  5%|██▋                                                 | 900/17200 [21:10<4:40:22,  1.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  6%|██▉                                                | 1000/17200 [23:15<4:29:55,  1.00it/s]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 4620
+  Batch size = 8
+  1%|▎                                                         | 3/578 [00:00<01:58,  4.83it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+100%|████████████████████████████████████████████████████████| 578/578 [02:20<00:00,  4.31it/s]
+Configuration saved in ./checkpoint-1000/config.json
+Model weights saved in ./checkpoint-1000/pytorch_model.bin
+Configuration saved in ./checkpoint-1000/preprocessor_config.json

wandb/run-20220109_220240-1g372i3v/logs/debug-internal.log

@@ -1765,3 +1765,543 @@
 2022-01-09 22:15:32,805 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
 2022-01-09 22:15:38,892 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
 2022-01-09 22:15:38,893 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:15:42,807 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:44,808 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:46,808 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:46,811 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:15:48,808 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:49,809 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/config.yaml
+2022-01-09 22:15:50,809 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:52,809 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:53,989 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:15:53,990 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:15:54,809 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:56,810 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:15:58,810 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:00,810 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:02,811 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:04,811 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:07,812 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:09,087 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:16:09,087 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:16:09,812 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:11,812 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:13,813 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:15,813 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:17,121 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:16:17,813 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:19,813 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:21,814 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:23,814 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:24,370 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:16:24,499 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:16:25,814 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:27,815 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:29,815 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:31,815 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:33,816 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:35,816 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:37,816 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:39,617 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:16:39,617 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:16:39,817 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:41,817 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:43,817 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:45,817 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:47,366 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:16:47,818 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:49,818 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:51,818 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:53,819 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:54,695 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:16:54,695 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:16:55,819 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:57,819 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:16:59,820 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:01,820 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:03,820 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:05,821 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:07,821 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:09,812 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:17:09,812 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:17:09,822 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:11,822 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:13,822 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:15,823 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:17,604 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:17:17,823 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:19,823 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:21,824 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:23,824 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:24,898 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:17:24,898 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:17:25,824 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:27,825 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:29,825 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:31,825 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:33,826 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:35,826 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:37,826 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:39,827 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:39,975 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:17:39,975 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:17:41,827 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:43,828 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:45,828 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:46,069 DEBUG   SenderThread:31514 [sender.py:send():182] send: history
+2022-01-09 22:17:46,124 DEBUG   SenderThread:31514 [sender.py:send():182] send: summary
+2022-01-09 22:17:46,147 INFO    SenderThread:31514 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-09 22:17:46,828 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/wandb-summary.json
+2022-01-09 22:17:47,828 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:47,854 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:17:48,829 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:49,829 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:51,829 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:53,830 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:55,093 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:17:55,093 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:17:55,830 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:57,831 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:17:59,831 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:01,831 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:03,831 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:05,832 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:07,832 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:09,833 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:10,194 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:18:10,194 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:18:11,833 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:13,834 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:15,834 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:17,835 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:18,093 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:18:19,835 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:22,835 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:24,836 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:25,265 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:18:25,266 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:18:26,836 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:28,837 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:30,837 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:32,837 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:34,838 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:36,838 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:38,838 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:40,336 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:18:40,337 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:18:40,839 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:42,839 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:44,839 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:46,840 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:48,343 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:18:48,840 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:50,840 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:52,841 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:54,841 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:55,531 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:18:55,532 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:18:56,841 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:18:58,842 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:00,842 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:02,842 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:04,843 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:06,843 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:08,843 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:10,610 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:19:10,610 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:19:10,844 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:12,844 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:14,844 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:16,845 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:18,599 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:19:18,845 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:20,845 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:22,845 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:24,846 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:25,679 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:19:25,679 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:19:26,846 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:28,846 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:30,847 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:32,847 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:34,847 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:38,848 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:40,769 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:19:40,769 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:19:40,848 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:42,849 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:44,849 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:46,849 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:48,845 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:19:48,849 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:50,142 DEBUG   SenderThread:31514 [sender.py:send():182] send: history
+2022-01-09 22:19:50,189 DEBUG   SenderThread:31514 [sender.py:send():182] send: summary
+2022-01-09 22:19:50,219 INFO    SenderThread:31514 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-09 22:19:50,850 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/wandb-summary.json
+2022-01-09 22:19:50,850 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:52,850 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:54,850 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:55,842 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:19:55,843 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:19:56,851 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:19:58,851 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:00,851 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:02,852 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:04,852 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:06,852 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:08,853 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:10,853 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:10,950 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:20:10,951 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:20:12,853 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:14,854 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:16,854 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:18,854 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:19,101 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:20:20,855 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:22,855 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:24,855 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:26,027 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:20:26,028 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:20:26,855 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:28,856 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:30,856 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:32,856 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:34,857 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:37,857 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:41,148 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:20:41,148 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:20:41,858 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:43,858 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:45,858 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:47,859 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:49,332 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:20:49,859 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:51,859 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:53,860 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:55,860 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:56,253 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:20:56,253 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:20:57,861 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:20:59,861 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:01,861 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:03,861 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:05,862 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:07,862 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:09,862 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:11,327 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:21:11,327 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:21:11,863 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:13,863 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:15,863 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:17,864 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:19,572 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:21:19,864 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:21,864 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:23,865 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:25,865 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:26,397 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:21:26,398 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:21:27,865 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:29,866 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:31,866 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:33,866 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:35,867 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:37,867 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:39,867 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:41,536 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:21:41,536 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:21:41,868 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:43,868 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:45,868 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:47,868 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:49,815 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:21:49,869 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:51,869 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:53,406 DEBUG   SenderThread:31514 [sender.py:send():182] send: history
+2022-01-09 22:21:53,453 DEBUG   SenderThread:31514 [sender.py:send():182] send: summary
+2022-01-09 22:21:53,481 INFO    SenderThread:31514 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-09 22:21:53,869 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/wandb-summary.json
+2022-01-09 22:21:53,870 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:55,870 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:56,731 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:21:56,731 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:21:57,870 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:21:59,870 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:01,871 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:03,871 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:05,871 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:07,872 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:09,872 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:11,806 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:22:11,806 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:22:11,872 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:13,873 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:15,873 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:17,873 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:19,874 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:20,058 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:22:21,874 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:23,874 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:25,875 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:26,881 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:22:26,881 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:22:27,875 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:29,875 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:31,875 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:33,876 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:35,876 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:37,876 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:39,877 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:41,877 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:41,953 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:22:41,953 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:22:43,877 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:45,878 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:48,878 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:50,307 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:22:50,878 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:52,879 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:54,879 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:56,879 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:22:57,051 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:22:57,052 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:22:58,880 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:00,880 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:02,880 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:04,881 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:06,881 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:08,881 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:10,882 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:12,162 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:23:12,162 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:23:12,882 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:14,882 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:16,883 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:18,883 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:20,555 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:23:20,883 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:22,884 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:24,884 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:26,884 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:27,312 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:23:27,312 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:23:28,885 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:30,885 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:32,885 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:34,886 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:36,886 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:38,886 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:40,887 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:42,384 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:23:42,384 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:23:42,887 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:46,888 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:48,888 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:50,799 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:23:50,888 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:52,889 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:54,889 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:56,889 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:57,455 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:23:57,455 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:23:57,547 DEBUG   SenderThread:31514 [sender.py:send():182] send: history
+2022-01-09 22:23:57,577 DEBUG   SenderThread:31514 [sender.py:send():182] send: summary
+2022-01-09 22:23:57,602 INFO    SenderThread:31514 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-09 22:23:57,889 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/wandb-summary.json
+2022-01-09 22:23:58,890 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:23:59,890 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:00,890 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:02,890 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:04,891 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:06,891 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:08,891 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:10,891 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:12,565 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:24:12,565 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:24:12,892 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:14,892 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:16,892 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:18,893 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:20,893 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:21,047 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:24:22,893 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:24,894 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:26,894 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:27,657 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:24:27,658 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:24:28,894 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:30,895 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:32,895 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:34,895 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:36,896 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:38,896 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:40,896 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:42,765 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:24:42,765 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:24:42,896 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:44,897 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:46,897 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:48,897 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:50,898 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:51,303 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:24:52,898 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:54,898 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:56,899 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:24:57,833 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:24:57,833 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:24:58,899 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:00,899 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:02,900 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:05,900 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:07,900 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:09,901 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:11,901 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:12,905 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:25:12,905 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:25:13,901 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:15,902 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:17,902 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:19,902 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:21,538 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:25:21,903 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:23,903 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:25,903 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:27,904 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:27,977 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:25:27,977 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:25:29,904 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:31,904 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:33,905 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:35,905 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:37,905 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:39,905 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:41,906 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:43,054 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:25:43,054 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:25:43,906 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:45,906 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:47,907 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:51,771 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:25:51,907 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:53,908 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:55,908 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:57,908 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:25:58,149 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:25:58,149 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:25:59,909 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:01,909 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:02,157 DEBUG   SenderThread:31514 [sender.py:send():182] send: history
+2022-01-09 22:26:02,205 DEBUG   SenderThread:31514 [sender.py:send():182] send: summary
+2022-01-09 22:26:02,231 INFO    SenderThread:31514 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-09 22:26:02,909 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/wandb-summary.json
+2022-01-09 22:26:03,909 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:05,910 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:07,910 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:09,910 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:11,911 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:13,338 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:26:13,338 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:26:13,911 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:15,911 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:17,912 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:19,912 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:21,912 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:22,020 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:26:23,912 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:25,913 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:27,913 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:28,425 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:26:28,425 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:26:29,913 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:31,914 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:33,914 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:35,914 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:37,915 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:39,915 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:41,915 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:43,501 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:26:43,501 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:26:43,916 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:45,916 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:47,916 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:49,917 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:51,917 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:52,272 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:26:53,917 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:55,918 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:57,918 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:26:58,610 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:26:58,610 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:26:59,918 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:01,918 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:04,919 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:06,919 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:08,920 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:10,920 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:12,920 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:13,742 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:27:13,742 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:27:14,920 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:16,921 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:18,921 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:20,921 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:22,519 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:27:22,922 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:24,922 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:26,922 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:28,825 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:27:28,826 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:27:28,923 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:30,923 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:32,923 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:34,924 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:36,924 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:38,924 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:40,925 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:42,925 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:43,911 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:27:43,911 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:27:44,925 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:46,926 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:48,926 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:50,926 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:52,761 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:27:52,927 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:54,927 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:56,927 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:58,928 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:27:58,979 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:27:58,979 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:28:00,928 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:02,928 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:04,929 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:06,929 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:08,929 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:10,930 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:12,930 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:14,072 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:28:14,072 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:28:14,930 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:16,931 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:18,931 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:20,931 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:22,932 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:23,009 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:28:24,500 DEBUG   SenderThread:31514 [sender.py:send():182] send: history
+2022-01-09 22:28:24,516 DEBUG   SenderThread:31514 [sender.py:send():182] send: summary
+2022-01-09 22:28:24,544 INFO    SenderThread:31514 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2022-01-09 22:28:24,932 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/wandb-summary.json
+2022-01-09 22:28:24,933 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:26,933 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:29,512 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:28:29,512 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:28:44,590 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:28:44,590 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:28:53,344 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats
+2022-01-09 22:28:56,948 INFO    Thread-8  :31514 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/hugging_face/examples/xls-r-300m-sv/wandb/run-20220109_220240-1g372i3v/files/output.log
+2022-01-09 22:28:59,668 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:28:59,668 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:29:14,744 DEBUG   HandlerThread:31514 [handler.py:handle_request():126] handle_request: stop_status
+2022-01-09 22:29:14,745 DEBUG   SenderThread:31514 [sender.py:send_request():196] send_request: stop_status
+2022-01-09 22:29:23,634 DEBUG   SenderThread:31514 [sender.py:send():182] send: stats