logs

events.out.tfevents.1626474829.t1v-n-f5c06ea1-w-0.798495.3.v2

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a8aebbd12cebb7e110121ed87c8f1b9dcdef4c391cbb90c97f8e37653c5d3d0
-size 1579452
+oid sha256:01467f333bc7718cb658947c162b9306af31f5f56506bcffd4a2ea25147222ba
+size 1774902

wandb/run-20210716_223350-8eukt20m/files/output.log

@@ -1644,3 +1644,190 @@ Training...: 10499it [1:16:12,  4.34it/s]█████████████
 
 
 
+
+Training...: 10549it [1:16:34,  4.92it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 10599it [1:16:54,  3.96it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 10649it [1:17:14,  5.33it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+Training...: 10700it [1:17:43,  3.05s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 10751it [1:18:03,  2.32s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 10802it [1:18:24,  1.69s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+Training...: 10850it [1:18:43,  3.09s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 10902it [1:19:04,  1.66s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 10952it [1:19:24,  1.67s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11002it [1:19:44,  1.61s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11053it [1:20:05,  1.20s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11102it [1:20:25,  1.67s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11152it [1:20:45,  1.69s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11199it [1:20:54,  4.83it/s]███████��█████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11249it [1:21:15,  5.82it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11299it [1:21:35,  4.16it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11349it [1:21:56,  5.32it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+
+Training...: 11401it [1:22:26,  1.36s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+Training...: 11453it [1:22:47,  1.15it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+Training...: 11499it [1:22:59,  5.78it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+Training...: 11550it [1:23:26,  2.21s/it]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+Training...: 11599it [1:23:36,  5.33it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+
+Training...: 11653it [1:24:07,  1.13it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+Training...: 11703it [1:24:28,  1.01it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+
+
+Training...: 11749it [1:24:42,  5.33it/s]█████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [01:17<00:00,  7.90it/s]
+
+
+
+
+
+

wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json

@@ -1 +1 @@
-{"training_step": 340500, "learning_rate": 2.2923233700566925e-05, "train_loss": 1.8997719287872314, "_runtime": 4935, "_timestamp": 1626479765, "_step": 210, "eval_step": 340000, "eval_accuracy": 0.6408576965332031, "eval_loss": 1.8213207721710205}
+{"training_step": 341800, "learning_rate": 2.2895399524713866e-05, "train_loss": 1.8234574794769287, "_runtime": 5460, "_timestamp": 1626480290, "_step": 236, "eval_step": 340000, "eval_accuracy": 0.6408576965332031, "eval_loss": 1.8213207721710205}

wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log

@@ -3240,3 +3240,370 @@
 2021-07-16 23:56:11,123 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
 2021-07-16 23:56:13,124 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
 2021-07-16 23:56:16,125 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:18,126 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:20,127 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:24,017 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:56:24,017 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:56:25,409 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:56:25,409 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:56:25,410 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:56:26,130 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:56:27,131 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:28,131 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:30,132 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:32,133 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:33,078 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:56:34,134 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:36,135 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:38,136 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:39,149 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:56:39,149 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:56:45,582 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:56:45,583 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:56:45,583 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:56:46,140 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:56:47,140 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:48,140 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:50,141 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:52,142 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:54,143 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:54,311 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:56:54,311 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:56:56,144 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:56:58,145 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:03,142 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:57:05,822 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:57:05,823 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:57:05,823 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:57:06,148 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:57:07,149 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:08,150 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:09,598 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:57:09,599 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:57:10,150 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:12,151 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:15,153 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:17,154 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:19,155 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:24,758 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:57:24,758 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:57:25,941 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:57:25,941 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:57:25,941 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:57:26,158 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:57:27,159 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:28,159 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:29,160 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:31,161 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:33,161 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:33,208 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:57:35,162 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:37,163 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:39,888 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:57:39,888 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:57:46,094 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:57:46,094 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:57:46,095 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:57:46,167 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:57:47,167 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:48,168 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:49,168 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:51,169 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:53,170 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:55,081 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:57:55,082 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:57:55,171 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:57:57,172 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:03,273 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:58:06,236 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:58:06,237 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:58:06,237 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:58:07,177 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:07,177 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:58:08,177 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:09,178 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:10,286 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:58:10,286 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:58:11,179 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:13,180 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:15,180 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:17,181 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:19,183 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:25,462 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:58:25,462 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:58:26,414 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:58:26,414 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:58:26,415 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:58:27,186 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:27,186 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:58:28,187 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:29,187 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:32,189 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:33,343 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:58:34,190 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:36,190 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:38,191 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:40,593 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:58:40,593 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:58:46,593 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:58:46,593 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:58:46,594 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:58:47,195 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:58:48,196 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:50,197 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:52,197 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:54,198 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:55,834 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:58:55,835 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:58:56,199 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:58:58,200 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:03,410 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:59:06,774 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:59:06,775 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:59:06,775 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:59:07,204 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:59:08,205 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:10,206 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:11,109 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:59:11,110 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:59:12,207 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:14,208 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:16,209 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:18,210 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:26,310 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:59:26,311 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:59:26,976 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:59:26,976 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:59:26,977 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:59:27,214 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:59:28,214 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:29,215 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:30,215 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:32,216 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:33,478 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-16 23:59:34,217 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:36,218 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:38,218 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:41,442 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:59:41,442 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:59:47,228 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-16 23:59:47,229 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-16 23:59:47,229 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-16 23:59:48,222 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:48,223 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-16 23:59:49,223 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:50,223 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:52,224 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:54,225 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:56,226 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-16 23:59:56,652 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-16 23:59:56,652 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-16 23:59:58,227 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:03,544 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:00:07,323 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:00:07,324 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:00:07,324 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:00:08,231 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:08,232 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:00:09,232 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:11,233 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:11,942 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:00:11,942 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:00:13,234 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:15,235 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:17,236 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:19,237 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:27,092 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:00:27,093 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:00:27,518 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:00:27,519 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:00:27,519 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:00:28,241 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:00:29,241 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:31,242 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:33,243 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:33,613 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:00:35,244 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:37,245 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:39,246 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:42,227 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:00:42,228 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:00:47,846 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:00:47,847 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:00:47,847 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:00:48,250 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:00:49,250 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:51,251 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:53,252 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:55,253 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:57,254 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:00:57,437 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:00:57,437 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:00:59,255 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:03,683 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:01:08,024 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:01:08,025 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:01:08,026 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:01:08,259 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:01:09,259 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:10,260 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:12,261 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:12,648 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:01:12,649 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:01:14,262 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:16,263 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:18,264 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:20,265 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:27,882 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:01:27,883 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:01:28,265 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:01:28,265 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:01:28,266 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:01:28,268 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:01:29,269 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:30,269 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:32,270 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:33,756 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:01:34,271 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:36,272 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:38,273 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:40,274 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:43,013 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:01:43,014 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:01:48,361 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:01:48,362 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:01:48,362 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:01:49,278 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:01:50,279 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:52,280 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:55,281 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:57,282 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:01:58,186 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:01:58,186 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:01:59,283 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:01,284 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:03,285 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:03,828 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:02:05,286 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:08,700 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:02:08,701 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:02:08,702 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:02:09,287 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:09,288 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:02:10,288 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:12,289 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:13,408 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:02:13,409 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:02:14,290 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:16,291 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:18,292 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:20,293 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:22,294 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:24,294 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:28,667 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:02:28,667 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:02:28,869 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:02:28,869 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:02:28,870 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:02:29,296 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:02:30,297 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:32,298 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:33,899 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:02:35,299 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:37,300 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:39,301 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:41,302 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:43,303 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:43,798 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:02:43,799 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:02:49,186 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:02:49,186 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:02:49,187 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:02:49,305 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:02:51,306 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:53,307 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:56,308 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:58,309 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:02:58,933 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:02:58,933 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:03:00,310 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:02,311 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:03,971 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:03:04,312 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:09,336 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:03:09,336 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:03:09,337 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:03:10,314 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:10,314 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:03:11,314 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:12,315 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:14,145 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:03:14,145 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:03:14,316 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:16,316 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:18,317 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:20,318 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:29,303 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:03:29,303 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:03:29,586 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:03:29,587 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:03:29,587 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:03:30,322 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:03:31,323 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:32,350 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:34,044 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:03:34,324 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:36,325 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:38,326 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:40,327 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:43,328 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:44,434 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:03:44,434 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:03:45,329 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:49,734 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:03:49,735 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:03:49,735 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:03:50,331 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:03:51,332 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:53,333 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:55,333 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:57,334 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:59,335 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:03:59,620 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:03:59,621 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:04:01,336 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:03,337 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:04,117 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:04:10,221 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:04:10,222 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:04:10,222 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:04:10,341 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:04:11,341 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:12,342 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:13,342 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:15,056 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:04:15,056 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:04:16,343 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:18,344 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:20,345 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:22,346 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:24,347 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:26,348 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:30,197 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:04:30,198 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:04:30,437 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:04:30,437 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:04:30,438 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end
+2021-07-17 00:04:31,350 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json
+2021-07-17 00:04:32,351 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:34,190 DEBUG   SenderThread:799749 [sender.py:send():179] send: stats
+2021-07-17 00:04:34,351 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:36,352 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:38,353 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:40,354 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:42,355 INFO    Thread-8  :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log
+2021-07-17 00:04:45,331 DEBUG   HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status
+2021-07-17 00:04:45,331 DEBUG   SenderThread:799749 [sender.py:send_request():193] send_request: stop_status
+2021-07-17 00:04:50,671 DEBUG   SenderThread:799749 [sender.py:send():179] send: history
+2021-07-17 00:04:50,671 DEBUG   SenderThread:799749 [sender.py:send():179] send: summary
+2021-07-17 00:04:50,671 INFO    SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end