diff --git "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" --- "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" +++ "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" @@ -12158,3 +12158,3754 @@ 2021-07-17 03:54:10,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log 2021-07-17 03:54:15,376 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status 2021-07-17 03:54:15,398 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:54:30,536 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:54:30,536 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:54:38,069 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:54:45,667 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:54:45,668 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:55:00,800 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:55:00,801 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:55:08,138 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:55:15,932 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:55:15,933 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:55:24,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:26,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:28,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:30,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:31,067 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:55:31,067 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:55:32,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:38,208 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:55:43,887 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:55:43,888 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:55:43,888 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:55:44,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:55:45,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:46,214 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:55:46,214 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:55:46,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:48,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:50,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:55:52,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:01,381 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:56:01,381 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:56:04,010 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:56:04,010 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:56:04,011 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:56:04,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:56:05,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:06,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:08,278 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:56:08,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:10,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:12,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:16,512 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:56:16,512 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:56:24,116 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:56:24,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:56:24,117 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:56:24,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:24,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:56:25,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:26,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:28,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:30,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:31,646 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:56:31,647 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:56:32,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:38,346 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:56:44,253 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:56:44,253 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:56:44,253 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:56:44,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:44,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:56:45,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:46,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:46,799 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:56:46,799 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:56:48,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:50,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:56:52,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:02,005 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:57:02,006 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:57:04,336 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:57:04,337 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:57:04,337 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:57:04,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:57:05,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:07,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:08,413 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:57:09,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:11,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:13,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:17,135 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:57:17,135 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:57:24,496 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:57:24,497 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:57:24,497 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:57:24,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:57:25,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:27,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:29,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:31,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:32,266 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:57:32,266 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:57:33,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:38,486 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:57:44,601 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:57:44,602 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:57:44,602 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:57:44,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:57:45,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:47,417 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:57:47,417 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:57:47,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:49,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:51,620 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:57:53,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:02,647 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:58:02,648 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:58:04,674 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:58:04,675 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:58:04,675 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:58:05,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:05,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:58:07,628 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:08,557 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:58:09,629 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:11,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:13,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:17,786 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:58:17,786 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:58:24,788 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:58:24,788 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:58:24,789 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:58:25,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:25,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:58:27,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:29,638 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:31,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:32,920 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:58:32,920 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:58:33,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:38,626 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:58:44,892 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:58:44,892 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:58:44,892 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:58:45,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:45,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:58:46,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:47,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:48,091 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:58:48,091 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:58:49,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:51,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:58:53,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:03,236 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:59:03,236 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:59:05,007 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:59:05,007 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:59:05,008 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:59:05,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:05,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:59:06,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:07,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:08,695 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:59:10,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:12,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:14,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:18,369 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:59:18,369 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:59:25,118 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:59:25,118 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:59:25,119 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:59:25,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:59:26,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:28,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:30,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:32,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:33,500 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:59:33,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:59:34,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:38,763 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 03:59:45,174 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 03:59:45,175 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 03:59:45,175 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 03:59:45,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 03:59:46,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:48,637 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 03:59:48,638 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 03:59:48,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:50,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:52,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 03:59:54,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:03,806 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:00:03,806 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:00:05,266 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:00:05,267 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:00:05,267 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:00:05,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:00:06,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:08,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:08,833 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:00:10,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:12,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:14,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:18,939 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:00:18,939 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:00:25,333 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:00:25,334 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:00:25,334 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:00:25,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:00:26,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:28,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:30,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:32,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:34,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:00:34,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:00:34,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:38,906 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:00:45,421 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:00:45,421 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:00:45,422 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:00:45,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:00:46,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:48,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:49,248 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:00:49,248 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:00:50,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:00:52,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:04,421 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:01:04,421 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:01:05,495 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:01:05,496 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:01:05,496 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:01:05,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:01:06,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:08,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:08,977 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:01:10,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:12,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:14,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:19,555 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:01:19,555 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:01:25,590 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:01:25,590 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:01:25,591 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:01:25,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:01:26,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:28,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:30,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:32,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:34,687 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:01:34,687 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:01:34,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:39,050 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:01:45,712 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:01:45,712 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:01:45,713 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:01:45,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:01:46,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:48,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:49,820 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:01:49,820 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:01:51,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:53,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:01:55,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:05,116 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:02:05,116 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:02:05,833 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:02:05,833 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:02:05,833 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:02:06,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:02:07,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:09,123 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:02:09,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:11,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:13,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:20,248 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:02:20,248 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:02:25,910 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:02:25,910 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:02:25,911 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:02:26,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:02:27,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:29,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:31,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:33,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:35,384 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:02:35,384 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:02:39,198 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:02:46,056 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:02:46,056 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:02:46,057 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:02:46,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:02:47,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:49,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:50,580 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:02:50,581 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:02:51,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:02:53,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:05,714 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:03:05,714 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:03:06,160 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:03:06,160 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:03:06,161 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:03:06,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:03:07,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:09,273 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:03:09,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:11,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:13,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:20,850 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:03:20,850 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:03:26,250 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:03:26,250 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:03:26,251 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:03:26,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:03:27,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:29,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:31,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:33,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:35,985 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:03:35,985 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:03:39,349 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:03:46,399 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:03:46,399 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:03:46,400 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:03:46,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:03:47,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:48,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:50,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:51,120 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:03:51,120 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:03:52,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:03:54,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:06,288 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:04:06,288 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:04:06,485 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:04:06,485 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:04:06,485 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:04:06,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:04:07,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:08,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:09,426 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:04:10,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:12,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:14,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:21,425 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:04:21,426 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:04:26,572 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:04:26,572 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:04:26,573 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:04:26,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:04:27,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:28,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:30,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:32,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:34,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:36,557 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:04:36,557 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:04:39,503 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:04:46,658 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:04:46,658 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:04:46,659 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:04:46,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:04:47,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:48,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:50,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:51,730 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:04:51,731 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:04:52,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:04:54,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:06,736 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:05:06,736 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:05:06,736 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:05:06,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:05:06,922 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:05:06,922 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:05:07,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:08,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:09,575 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:05:10,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:12,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:14,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:22,113 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:05:22,114 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:05:26,831 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:05:26,832 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:05:26,832 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:05:26,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:05:27,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:28,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:30,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:32,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:34,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:37,244 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:05:37,245 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:05:39,651 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:05:46,985 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:05:46,986 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:05:46,986 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:05:47,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:05:48,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:50,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:52,412 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:05:52,412 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:05:53,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:05:55,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:07,097 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:06:07,097 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:06:07,097 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:06:07,764 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:06:07,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:06:07,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:06:08,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:09,725 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:06:09,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:11,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:13,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:15,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:22,917 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:06:22,918 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:06:27,218 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:06:27,219 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:06:27,220 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:06:27,872 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:06:28,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:29,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:31,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:33,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:35,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:38,053 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:06:38,054 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:06:39,806 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:06:47,409 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:06:47,410 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:06:47,410 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:06:47,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:06:48,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:49,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:51,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:53,185 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:06:53,186 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:06:53,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:06:55,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:07,472 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:07:07,473 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:07:07,473 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:07:07,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:07:08,323 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:07:08,324 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:07:08,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:09,882 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:07:09,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:11,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:13,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:15,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:23,548 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:07:23,548 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:07:27,533 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:07:27,534 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:07:27,535 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:07:27,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:07:28,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:29,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:31,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:33,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:35,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:38,679 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:07:38,679 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:07:39,962 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:07:47,592 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:07:47,592 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:07:47,593 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:07:47,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:07:48,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:49,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:51,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:53,907 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:07:53,908 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:07:53,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:07:55,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:07,708 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:08:07,708 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:08:07,709 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:08:07,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:08:08,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:09,051 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:08:09,051 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:08:09,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:10,036 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:08:11,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:13,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:15,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:24,208 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:08:24,208 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:08:27,825 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:08:27,825 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:08:27,826 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:08:27,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:08:28,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:29,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:31,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:33,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:35,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:39,343 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:08:39,343 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:08:40,110 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:08:47,930 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:08:47,930 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:08:47,931 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:08:47,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:08:49,942 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:51,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:53,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:08:54,474 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:08:54,474 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:08:55,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:07,988 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:09:07,988 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:09:07,989 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:09:08,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:09:09,640 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:09:09,641 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:09:09,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:10,184 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:09:11,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:13,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:15,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:24,792 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:09:24,792 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:09:28,038 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:09:28,039 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:09:28,039 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:09:28,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:09:29,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:31,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:33,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:35,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:39,921 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:09:39,922 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:09:40,258 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:09:48,095 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:09:48,095 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:09:48,096 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:09:48,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:09:49,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:52,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:54,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:09:55,262 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:09:55,263 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:09:56,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:08,184 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:10:08,185 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:10:08,185 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:10:08,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:10:09,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:10,331 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:10:10,457 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:10:10,457 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:10:10,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:12,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:14,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:16,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:25,588 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:10:25,588 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:10:28,244 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:10:28,244 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:10:28,245 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:10:28,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:10:29,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:30,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:32,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:34,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:36,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:40,406 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:10:40,719 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:10:40,719 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:10:48,326 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:10:48,326 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:10:48,326 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:10:48,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:10:50,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:51,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:53,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:55,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:10:55,849 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:10:55,849 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:10:57,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:08,419 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:11:08,420 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:11:08,420 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:11:09,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:11:10,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:10,480 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:11:11,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:11,027 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:11:11,027 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:11:13,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:15,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:17,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:26,160 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:11:26,160 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:11:28,551 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:11:28,552 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:11:28,552 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:11:29,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:11:30,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:31,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:33,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:35,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:37,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:40,555 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:11:41,290 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:11:41,291 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:11:48,654 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:11:48,655 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:11:48,655 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:11:49,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:11:50,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:51,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:53,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:55,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:11:56,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:11:56,422 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:11:57,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:08,779 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:12:08,779 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:12:08,780 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:12:09,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:12:10,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:10,629 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:12:11,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:11,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:12:11,566 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:12:13,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:15,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:17,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:26,702 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:12:26,702 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:12:28,919 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:12:28,919 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:12:28,920 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:12:29,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:12:30,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:31,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:33,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:35,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:37,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:40,703 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:12:41,832 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:12:41,832 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:12:49,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:12:49,022 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:12:49,022 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:12:49,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:12:50,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:51,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:53,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:55,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:12:56,967 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:12:56,967 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:12:57,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:09,109 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:13:09,110 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:13:09,110 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:13:10,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:13:10,779 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:13:11,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:12,107 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:13:12,108 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:13:13,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:15,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:17,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:27,259 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:13:27,260 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:13:29,206 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:13:29,206 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:13:29,206 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:13:30,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:13:31,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:33,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:35,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:37,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:40,856 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:13:42,391 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:13:42,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:13:49,302 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:13:49,302 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:13:49,303 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:13:50,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:13:51,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:53,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:56,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:13:57,521 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:13:57,522 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:13:58,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:09,391 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:14:09,392 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:14:09,392 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:14:10,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:14:10,930 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:14:11,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:12,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:12,651 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:14:12,652 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:14:14,095 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:16,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:18,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:28,018 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:14:28,019 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:14:29,541 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:14:29,541 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:14:29,542 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:14:30,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:14:31,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:32,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:34,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:36,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:38,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:41,007 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:14:43,149 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:14:43,149 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:14:49,603 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:14:49,604 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:14:49,604 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:14:50,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:14:51,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:52,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:54,115 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:56,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:58,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:14:58,280 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:14:58,280 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:15:09,671 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:15:09,671 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:15:09,672 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:15:10,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:15:11,082 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:15:11,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:12,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:13,456 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:15:13,457 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:15:14,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:16,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:18,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:28,654 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:15:28,654 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:15:29,816 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:15:29,816 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:15:29,817 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:15:30,131 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:15:31,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:32,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:34,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:36,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:38,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:41,159 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:15:43,790 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:15:43,790 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:15:49,898 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:15:49,899 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:15:49,899 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:15:50,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:15:51,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:52,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:54,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:56,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:58,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:15:58,920 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:15:58,920 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:16:10,017 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:16:10,017 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:16:10,018 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:16:10,148 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:16:11,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:11,235 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:16:12,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:14,121 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:16:14,121 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:16:14,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:16,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:18,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:29,253 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:16:29,254 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:16:30,128 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:16:30,128 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:16:30,129 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:16:30,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:16:31,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:32,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:34,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:36,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:38,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:41,310 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:16:44,384 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:16:44,385 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:16:50,207 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:16:50,207 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:16:50,208 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:16:51,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:51,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:16:52,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:54,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:56,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:58,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:16:59,517 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:16:59,517 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:17:10,336 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:17:10,336 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:17:10,337 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:17:11,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:11,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:17:11,387 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:17:13,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:14,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:17:14,662 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:17:15,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:17,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:19,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:29,818 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:17:29,819 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:17:30,461 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:17:30,461 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:17:30,462 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:17:31,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:31,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:17:33,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:35,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:37,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:39,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:41,465 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:17:44,951 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:17:44,952 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:17:50,517 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:17:50,517 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:17:50,518 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:17:51,193 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:17:52,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:53,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:55,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:57,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:17:59,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:00,114 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:18:00,114 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:18:10,614 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:18:10,615 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:18:10,615 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:18:11,202 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:18:11,542 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:18:12,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:13,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:15,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:15,478 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:18:15,479 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:18:17,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:19,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:30,638 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:18:30,638 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:18:30,768 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:18:30,768 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:18:30,768 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:18:31,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:18:32,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:33,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:35,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:37,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:39,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:41,619 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:18:45,782 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:18:45,782 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:18:50,780 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:18:50,781 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:18:50,781 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:18:51,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:18:52,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:53,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:55,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:57,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:18:59,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:00,915 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:19:00,915 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:19:10,862 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:19:10,862 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:19:10,863 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:19:11,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:19:11,699 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:19:12,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:13,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:15,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:16,098 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:19:16,098 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:19:17,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:19,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:30,993 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:19:30,994 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:19:30,994 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:19:31,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:19:31,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:19:31,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:19:32,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:33,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:35,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:37,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:39,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:41,776 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:19:46,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:19:46,460 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:19:51,056 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:19:51,056 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:19:51,056 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:19:51,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:19:52,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:53,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:55,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:57,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:19:59,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:01,592 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:20:01,592 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:20:11,136 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:20:11,136 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:20:11,137 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:20:11,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:20:11,853 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:20:12,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:13,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:15,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:16,746 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:20:16,746 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:20:17,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:19,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:31,254 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:20:31,254 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:20:31,254 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:20:31,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:20:31,935 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:20:31,935 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:20:32,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:33,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:35,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:37,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:39,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:41,930 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:20:47,078 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:20:47,079 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:20:51,372 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:20:51,373 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:20:51,373 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:20:52,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:20:53,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:55,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:57,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:20:59,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:02,209 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:21:02,209 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:21:11,493 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:21:11,493 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:21:11,494 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:21:12,004 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:21:12,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:21:13,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:14,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:16,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:17,343 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:21:17,343 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:21:18,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:20,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:31,609 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:21:31,610 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:21:31,611 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:21:32,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:21:32,531 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:21:32,531 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:21:33,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:34,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:36,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:38,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:40,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:42,081 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:21:47,668 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:21:47,668 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:21:51,678 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:21:51,679 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:21:51,679 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:21:52,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:21:53,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:54,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:56,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:21:58,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:00,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:02,799 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:22:02,800 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:22:11,762 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:22:11,762 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:22:11,763 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:22:12,158 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:22:12,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:22:13,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:14,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:16,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:17,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:22:17,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:22:18,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:20,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:31,860 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:22:31,860 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:22:31,861 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:22:32,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:22:33,165 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:22:33,166 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:22:33,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:34,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:36,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:38,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:40,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:42,236 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:22:48,372 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:22:48,373 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:22:51,979 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:22:51,979 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:22:51,980 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:22:52,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:22:53,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:54,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:56,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:22:58,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:00,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:03,502 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:23:03,502 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:23:12,065 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:23:12,066 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:23:12,066 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:23:12,313 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:23:12,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:23:13,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:14,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:16,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:18,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:18,638 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:23:18,639 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:23:20,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:32,168 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:23:32,168 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:23:32,169 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:23:32,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:23:33,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:33,877 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:23:33,878 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:23:34,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:36,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:38,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:40,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:42,391 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:23:49,015 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:23:49,016 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:23:52,235 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:23:52,235 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:23:52,235 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:23:52,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:23:54,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:56,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:23:58,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:00,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:04,148 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:24:04,148 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:24:12,292 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:24:12,293 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:24:12,293 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:24:12,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:24:12,468 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:24:14,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:16,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:18,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:19,278 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:24:19,278 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:24:20,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:32,342 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:24:32,342 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:24:32,342 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:24:32,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:24:34,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:34,411 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:24:34,411 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:24:36,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:38,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:40,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:42,543 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:24:49,544 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:24:49,545 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:24:52,473 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:24:52,473 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:24:52,474 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:24:53,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:24:54,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:56,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:24:58,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:00,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:04,677 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:25:04,677 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:25:12,578 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:25:12,578 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:25:12,581 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:25:12,619 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:25:13,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:25:14,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:15,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:17,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:19,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:19,810 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:25:19,810 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:25:21,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:32,632 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:25:32,632 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:25:32,633 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:25:33,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:25:34,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:34,945 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:25:34,945 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:25:35,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:37,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:39,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:41,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:42,698 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:25:50,078 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:25:50,079 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:25:52,696 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:25:52,697 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:25:52,697 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:25:53,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:25:54,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:55,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:57,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:25:59,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:01,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:05,210 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:26:05,210 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:26:12,745 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:26:12,746 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:26:12,748 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:26:12,775 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:26:13,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:26:14,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:15,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:17,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:19,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:20,347 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:26:20,347 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:26:21,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:32,825 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:26:32,826 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:26:32,826 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:26:33,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:26:34,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:35,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:35,508 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:26:35,509 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:26:37,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:39,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:41,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:42,855 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:26:50,709 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:26:50,709 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:26:52,928 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:26:52,929 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:26:52,929 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:26:53,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:26:54,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:55,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:57,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:26:59,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:01,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:05,842 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:27:05,842 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:27:12,931 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:27:13,024 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:27:13,025 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:27:13,025 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:27:13,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:27:14,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:15,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:17,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:19,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:20,973 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:27:20,973 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:27:21,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:33,077 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:27:33,078 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:27:33,078 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:27:33,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:27:34,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:35,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:36,158 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:27:36,158 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:27:37,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:39,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:41,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:43,004 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:27:51,356 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:27:51,357 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:27:53,143 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:27:53,143 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:27:53,144 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:27:53,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:27:54,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:55,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:57,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:27:59,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:01,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:06,487 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:28:06,487 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:28:13,078 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:28:13,212 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:28:13,212 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:28:13,212 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:28:13,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:28:14,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:15,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:17,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:19,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:21,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:21,619 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:28:21,620 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:28:33,317 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:28:33,317 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:28:33,318 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:28:33,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:33,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:28:34,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:36,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:36,764 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:28:36,764 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:28:38,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:40,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:42,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:43,154 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:28:51,915 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:28:51,915 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:28:53,360 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:28:53,361 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:28:53,361 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:28:53,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:28:54,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:56,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:28:58,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:00,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:02,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:04,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:06,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:07,051 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:29:07,051 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:29:08,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:10,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:12,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:13,221 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:29:14,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:16,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:18,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:20,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:22,184 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:29:22,185 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:29:22,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:24,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:26,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:28,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:30,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:32,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:34,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:36,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:37,343 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:29:37,343 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:29:38,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:40,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:42,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:43,287 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:29:44,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:46,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:48,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:50,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:52,473 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:29:52,474 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:29:52,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:54,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:29:57,223 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:29:57,224 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:29:57,224 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:29:57,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:29:58,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:00,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:02,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:04,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:06,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:07,605 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:30:07,605 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:30:13,353 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:30:17,437 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:30:17,437 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:30:17,438 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:30:17,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:30:18,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:19,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:20,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:22,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:22,783 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:30:22,783 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:30:24,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:26,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:37,497 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:30:37,497 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:30:37,498 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:30:37,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:30:37,918 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:30:37,919 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:30:38,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:39,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:40,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:43,422 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:30:43,532 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:45,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:30:53,111 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:30:53,111 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:30:57,568 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:30:57,569 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:30:57,569 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:30:58,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:30:59,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:01,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:03,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:05,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:07,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:08,244 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:31:08,244 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:31:13,490 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:31:17,919 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:31:17,919 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:31:17,919 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:31:18,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:31:19,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:21,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:23,374 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:31:23,374 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:31:23,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:25,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:27,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:38,003 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:31:38,004 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:31:38,004 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:31:38,505 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:31:38,506 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:31:38,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:31:39,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:41,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:43,556 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:31:43,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:45,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:31:53,697 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:31:53,698 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:31:58,067 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:31:58,067 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:31:58,068 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:31:58,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:31:59,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:01,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:03,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:05,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:07,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:08,830 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:32:08,831 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:32:13,623 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:32:18,152 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:32:18,153 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:32:18,153 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:32:18,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:32:19,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:21,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:23,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:23,981 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:32:23,982 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:32:25,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:27,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:38,261 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:32:38,262 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:32:38,262 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:32:38,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:32:39,312 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:32:39,312 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:32:39,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:41,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:43,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:43,691 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:32:45,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:47,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:32:54,459 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:32:54,459 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:32:58,338 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:32:58,339 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:32:58,339 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:32:58,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:32:59,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:00,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:01,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:03,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:05,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:09,592 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:33:09,593 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:33:13,759 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:33:18,450 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:33:18,451 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:33:18,451 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:33:18,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:33:19,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:20,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:21,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:23,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:24,724 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:33:24,724 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:33:25,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:27,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:38,523 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:33:38,524 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:33:38,524 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:33:38,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:33:39,968 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:33:39,968 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:33:40,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:42,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:43,829 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:33:44,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:46,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:48,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:33:55,107 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:33:55,107 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:33:58,627 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:33:58,628 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:33:58,628 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:33:59,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:34:00,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:02,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:04,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:06,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:08,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:10,237 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:34:10,238 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:34:13,900 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:34:18,733 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:34:18,733 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:34:18,734 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:34:19,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:34:20,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:22,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:24,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:25,370 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:34:25,371 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:34:26,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:38,780 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:34:38,781 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:34:38,781 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:34:39,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:34:40,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:34:40,567 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:34:40,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:42,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:43,973 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:34:44,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:46,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:34:55,736 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:34:55,736 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:34:58,908 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:34:58,908 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:34:58,909 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:34:59,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:35:00,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:02,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:04,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:06,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:08,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:10,867 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:35:10,868 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:35:14,045 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:35:18,963 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:35:18,964 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:35:18,964 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:35:19,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:35:20,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:22,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:24,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:25,998 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:35:25,999 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:35:26,654 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:39,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:35:39,022 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:35:39,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:35:39,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:35:40,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:41,179 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:35:41,180 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:35:42,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:44,119 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:35:44,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:46,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:35:56,369 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:35:56,369 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:35:59,087 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:35:59,087 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:35:59,088 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:35:59,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:36:00,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:02,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:04,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:06,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:11,502 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:36:11,503 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:36:14,193 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:36:19,161 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:36:19,161 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:36:19,162 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:36:19,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:36:20,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:22,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:24,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:26,634 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:36:26,635 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:36:27,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:39,242 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:36:39,242 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:36:39,243 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:36:39,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:36:41,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:41,788 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:36:41,788 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:36:43,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:44,267 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:36:45,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:47,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:36:56,923 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:36:56,923 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:36:59,310 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:36:59,311 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:36:59,311 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:36:59,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:37:01,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:03,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:05,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:07,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:12,055 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:37:12,055 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:37:14,336 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:37:19,369 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:37:19,369 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:37:19,370 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:37:19,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:37:21,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:23,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:25,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:27,185 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:37:27,185 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:37:27,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:39,482 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:37:39,482 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:37:39,483 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:37:39,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:37:41,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:42,316 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:37:42,316 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:37:43,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:44,408 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:37:45,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:47,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:37:57,452 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:37:57,452 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:37:59,550 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:37:59,550 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:37:59,551 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:37:59,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:38:01,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:03,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:05,722 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:07,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:12,584 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:38:12,584 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:38:14,480 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:38:19,662 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:38:19,663 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:38:19,663 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:38:19,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:38:21,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:23,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:25,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:27,715 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:38:27,715 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:38:27,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:39,717 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:38:39,717 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:38:39,718 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:38:39,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:38:41,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:42,889 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:38:42,889 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:38:43,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:44,556 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:38:45,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:47,740 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:38:58,025 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:38:58,025 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:38:59,799 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:38:59,799 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:38:59,799 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:39:00,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:39:01,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:03,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:05,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:07,749 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:13,157 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:39:13,158 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:39:14,630 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:39:19,850 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:39:19,850 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:39:19,851 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:39:20,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:39:21,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:23,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:25,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:27,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:28,288 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:39:28,289 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:39:39,961 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:39:39,961 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:39:39,962 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:39:40,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:39:41,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:43,429 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:39:43,430 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:39:43,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:44,702 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:39:45,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:47,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:39:58,570 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:39:58,571 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:40:00,100 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:40:00,100 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:40:00,101 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:40:00,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:40:01,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:03,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:05,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:07,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:13,701 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:40:13,702 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:40:14,777 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:40:20,236 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:40:20,236 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:40:20,237 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:40:20,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:40:21,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:23,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:25,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:27,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:28,831 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:40:28,832 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:40:40,313 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:40:40,313 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:40:40,314 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:40:40,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:40:41,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:42,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:43,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:44,043 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:40:44,043 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:40:44,853 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:40:45,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:48,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:40:59,215 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:40:59,215 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:41:00,472 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:41:00,472 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:41:00,472 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:41:00,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:41:02,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:04,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:06,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:08,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:14,347 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:41:14,347 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:41:14,926 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:41:20,548 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:41:20,549 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:41:20,549 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:41:20,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:41:22,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:24,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:26,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:28,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:29,477 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:41:29,478 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:41:40,694 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:41:40,694 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:41:40,695 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:41:40,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:41:42,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:44,668 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:41:44,668 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:41:44,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:45,001 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:41:46,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:48,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:41:59,853 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:41:59,853 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:42:00,758 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:42:00,758 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:42:00,759 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:42:00,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:42:02,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:04,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:06,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:08,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:14,989 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:42:14,989 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:42:15,125 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:42:20,826 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:42:20,826 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:42:20,826 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:42:20,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:42:22,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:24,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:26,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:28,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:30,127 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:42:30,128 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:42:40,895 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:42:40,895 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:42:40,896 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:42:41,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:42:42,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:44,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:45,149 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:42:45,280 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:42:45,280 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:42:46,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:42:48,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:00,410 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:43:00,411 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:43:00,979 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:43:00,979 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:43:00,979 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:43:01,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:43:02,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:04,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:06,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:08,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:15,220 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:43:15,543 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:43:15,543 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:43:21,015 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:43:21,015 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:43:21,016 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:43:21,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:43:22,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:24,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:26,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:28,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:30,673 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:43:30,674 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:43:41,075 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:43:41,076 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:43:41,076 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:43:41,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:43:42,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:44,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:45,294 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:43:45,845 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:43:45,845 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:43:46,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:43:48,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:01,014 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:44:01,014 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:44:01,204 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:44:01,204 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:44:01,205 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:44:01,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:44:02,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:04,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:06,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:08,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:15,368 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:44:16,147 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:44:16,147 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:44:21,302 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:44:21,303 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:44:21,304 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:44:21,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:44:22,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:24,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:26,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:28,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:31,279 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:44:31,279 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:44:41,388 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:44:41,389 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:44:41,389 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:44:41,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:44:42,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:44,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:45,444 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:44:46,495 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:44:46,495 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:44:47,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:44:49,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:01,494 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:45:01,494 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:45:01,495 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:45:01,684 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:45:01,684 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:45:01,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:45:02,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:03,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:05,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:07,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:09,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:15,518 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:45:16,871 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:45:16,871 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:45:21,574 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:45:21,574 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:45:21,574 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:45:21,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:45:22,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:23,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:25,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:27,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:29,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:32,009 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:45:32,009 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:45:41,624 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:45:41,625 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:45:41,625 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:45:41,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:45:42,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:43,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:45,593 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:45:45,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:47,214 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:45:47,215 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:45:47,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:45:49,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:01,684 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:46:01,684 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:46:01,685 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:46:01,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:46:02,401 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:46:02,401 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:46:03,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:05,942 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:07,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:09,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:15,666 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:46:17,558 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:46:17,558 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:46:21,784 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:46:21,785 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:46:21,785 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:46:21,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:46:23,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:25,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:27,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:29,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:32,696 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:46:32,697 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:46:41,833 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:46:41,833 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:46:41,834 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:46:41,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:46:43,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:45,738 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:46:45,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:47,829 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:46:47,830 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:46:47,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:46:49,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:01,890 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:47:01,891 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:47:01,891 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:47:01,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:47:03,017 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:47:03,018 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:47:03,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:05,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:07,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:09,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:15,814 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:47:18,180 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:47:18,180 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:47:22,046 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:47:22,047 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:47:22,047 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:47:22,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:47:23,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:25,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:27,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:29,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:33,311 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:47:33,312 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:47:42,354 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:47:42,354 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:47:42,355 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:47:42,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:47:43,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:45,890 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:47:45,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:47,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:47:48,442 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:47:48,442 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:47:49,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:02,477 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:48:02,477 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:48:02,477 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:48:02,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:48:03,628 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:48:03,628 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:48:03,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:04,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:06,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:08,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:10,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:15,966 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:48:18,764 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:48:18,764 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:48:22,593 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:48:22,593 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:48:22,594 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:48:23,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:48:24,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:25,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:27,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:29,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:31,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:33,896 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:48:33,896 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:48:42,736 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:48:42,736 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:48:42,737 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:48:43,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:48:44,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:45,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:46,041 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:48:47,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:49,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:48:49,029 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:48:49,030 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:48:51,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:02,795 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:49:02,796 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:49:02,796 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:49:03,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:49:04,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:04,164 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:49:04,165 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:49:05,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:07,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:09,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:11,025 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:16,115 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:49:19,341 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:49:19,341 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:49:22,869 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:49:22,870 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:49:22,870 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:49:23,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:49:24,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:25,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:27,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:29,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:31,034 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:34,473 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:49:34,473 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:49:42,915 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:49:42,915 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:49:42,916 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:49:43,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:49:44,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:46,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:46,191 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:49:48,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:49,609 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:49:49,609 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:49:50,043 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:49:52,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:03,034 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:50:03,034 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:50:03,035 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:50:03,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:50:04,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:04,782 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:50:04,783 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:50:06,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:08,052 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:10,053 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:12,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:16,266 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:50:19,926 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:50:19,926 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:50:23,096 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:50:23,096 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:50:23,097 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:50:24,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:24,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:50:25,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:26,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:28,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:30,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:32,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:35,057 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:50:35,057 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:50:43,197 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:50:43,197 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:50:43,198 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:50:44,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:44,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:50:45,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:46,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:46,339 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:50:48,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:50,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:50:50,191 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:50:50,191 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:50:52,071 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:03,289 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:51:03,289 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:51:03,290 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:51:04,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:04,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:51:05,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:05,367 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:51:05,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:51:06,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:08,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:10,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:12,080 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:16,415 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:51:20,500 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:51:20,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:51:23,339 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:51:23,340 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:51:23,340 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:51:24,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:51:25,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:26,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:28,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:30,089 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:32,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:35,632 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:51:35,632 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:51:43,395 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:51:43,395 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:51:43,396 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:51:44,095 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:44,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:51:45,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:46,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:46,491 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:51:48,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:50,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:51:50,772 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:51:50,773 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:51:52,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:03,474 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:52:03,474 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:52:03,475 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:52:04,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:04,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:52:05,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:05,940 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:52:05,941 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:52:06,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:08,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:10,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:12,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:16,564 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:52:21,102 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:52:21,103 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:52:23,523 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:52:23,524 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:52:23,524 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:52:24,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:24,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:52:25,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:27,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:29,115 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:31,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:33,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:36,235 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:52:36,236 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:52:43,572 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:52:43,572 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:52:43,573 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:52:44,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:52:45,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:46,637 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:52:47,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:49,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:51,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:52:51,367 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:52:51,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:52:53,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:03,652 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:53:03,653 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:53:03,654 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:53:04,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:53:05,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:06,511 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:53:06,511 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:53:07,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:09,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:11,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:13,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:16,711 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:53:21,660 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:53:21,660 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:53:23,713 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:53:23,714 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:53:23,714 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:53:24,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:53:25,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:27,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:29,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:31,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:36,790 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:53:36,790 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:53:43,819 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:53:43,819 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:53:43,820 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:53:44,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:53:45,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:46,787 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:53:47,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:49,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:51,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:53:51,927 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:53:51,928 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:53:53,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:03,875 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:54:03,875 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:54:03,876 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:54:04,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:54:05,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:06,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:07,113 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:54:07,114 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:54:07,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:09,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:11,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:13,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:16,860 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:54:22,255 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:54:22,256 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:54:23,920 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:54:23,921 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:54:23,921 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:54:24,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:54:25,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:26,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:27,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:29,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:31,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:33,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:37,386 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:54:37,386 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:54:44,050 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:54:44,050 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:54:44,051 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:54:44,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:54:45,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:46,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:46,934 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:54:47,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:49,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:51,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:54:52,520 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:54:52,521 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:54:53,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:04,135 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:55:04,136 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:55:04,136 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:55:04,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:55:06,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:07,675 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:55:07,675 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:55:08,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:10,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:12,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:14,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:17,009 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:55:22,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:55:22,814 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:55:24,191 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:55:24,191 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:55:24,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:55:24,193 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:55:26,193 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:28,194 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:30,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:32,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:37,943 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:55:37,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:55:44,302 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:55:44,302 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:55:44,303 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:55:45,202 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:55:46,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:47,086 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:55:48,203 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:50,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:52,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:55:53,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:55:53,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:56:04,398 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:56:04,399 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:56:04,399 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:56:05,211 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:56:06,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:08,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:08,213 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:56:08,213 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:56:10,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:12,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:14,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:17,163 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:56:23,378 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:56:23,379 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:56:24,460 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:56:24,461 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:56:24,461 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:56:25,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:56:26,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:28,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:30,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:32,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:34,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:38,510 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:56:38,510 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:56:44,582 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:56:44,582 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:56:44,583 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:56:45,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:56:46,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:47,238 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:56:48,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:50,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:52,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:56:53,644 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:56:53,644 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:57:04,682 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:57:04,682 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:57:04,683 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:57:05,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:57:06,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:08,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:08,819 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:57:08,820 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:57:10,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:12,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:17,312 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:57:24,001 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:57:24,001 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:57:24,832 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:57:24,832 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:57:24,833 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:57:25,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:57:26,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:28,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:30,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:32,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:39,141 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:57:39,142 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:57:44,886 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:57:44,887 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:57:44,887 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:57:45,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:57:46,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:47,387 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:57:48,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:50,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:52,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:57:54,282 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:57:54,282 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:58:04,961 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:58:04,961 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:58:04,962 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:58:05,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:58:06,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:09,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:09,459 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:58:09,459 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:58:11,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:13,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:17,466 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:58:24,633 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:58:24,634 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:58:25,037 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:58:25,038 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:58:25,038 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:58:25,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:58:26,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:27,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:29,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:31,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:33,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:39,764 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:58:39,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:58:45,093 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:58:45,094 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:58:45,094 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:58:45,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:58:46,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:47,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:47,541 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:58:49,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:51,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:53,289 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:58:54,897 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:58:54,898 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:59:05,212 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:59:05,212 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:59:05,213 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:59:05,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:59:06,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:07,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:09,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:10,040 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:59:10,040 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:59:11,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:13,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:17,616 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:59:25,181 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:59:25,182 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:59:25,359 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:59:25,360 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:59:25,360 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:59:26,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:59:27,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:29,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:31,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:33,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:40,312 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:59:40,312 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 04:59:45,421 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 04:59:45,422 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 04:59:45,422 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 04:59:46,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 04:59:47,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:47,692 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 04:59:49,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:51,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:53,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 04:59:55,444 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 04:59:55,444 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:00:05,516 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:00:05,516 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:00:05,517 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:00:06,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:00:07,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:09,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:10,594 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:00:10,595 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:00:11,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:13,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:17,767 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:00:25,579 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:00:25,580 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:00:25,580 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:00:25,791 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:00:25,791 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:00:26,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:00:27,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:29,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:31,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:33,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:40,929 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:00:40,929 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:00:45,640 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:00:45,640 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:00:45,641 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:00:46,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:00:47,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:47,843 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:00:49,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:51,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:53,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:00:56,060 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:00:56,060 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:01:05,710 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:01:05,710 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:01:05,711 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:01:06,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:01:07,356 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:10,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:11,190 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:01:11,191 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:01:12,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:14,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:17,913 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:01:25,786 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:01:25,786 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:01:25,787 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:01:26,330 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:01:26,330 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:01:26,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:01:27,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:28,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:30,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:32,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:34,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:41,462 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:01:41,463 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:01:45,863 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:01:45,864 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:01:45,864 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:01:46,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:01:47,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:47,981 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:01:48,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:50,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:52,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:54,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:01:56,604 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:01:56,604 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:02:05,963 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:02:05,963 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:02:05,964 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:02:06,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:02:07,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:08,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:10,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:11,879 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:02:11,879 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:02:12,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:14,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:18,049 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:02:26,061 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:02:26,062 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:02:26,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:02:26,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:02:27,010 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:02:27,010 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:02:27,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:28,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:30,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:32,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:34,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:42,146 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:02:42,146 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:02:46,130 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:02:46,130 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:02:46,130 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:02:46,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:02:47,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:48,116 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:02:48,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:50,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:52,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:54,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:02:57,278 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:02:57,278 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:03:06,257 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:03:06,257 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:03:06,258 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:03:06,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:03:07,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:08,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:10,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:12,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:12,455 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:03:12,455 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:03:14,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:18,187 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:03:26,307 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:03:26,307 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:03:26,308 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:03:26,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:03:27,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:27,586 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:03:27,586 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:03:28,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:30,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:32,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:34,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:42,765 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:03:42,766 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:03:46,430 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:03:46,430 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:03:46,431 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:03:46,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:03:47,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:48,258 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:03:48,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:50,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:52,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:54,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:03:57,895 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:03:57,895 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:04:06,714 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:04:06,715 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:04:06,715 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:04:07,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:07,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:04:08,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:10,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:13,029 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:04:13,030 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:04:13,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:15,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:18,330 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:04:26,794 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:04:26,794 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:04:26,795 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:04:27,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:04:28,230 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:04:28,230 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:04:28,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:29,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:31,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:33,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:35,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:43,376 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:04:43,376 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:04:46,842 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:04:46,842 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:04:46,842 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:04:47,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:04:48,399 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:04:48,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:49,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:51,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:53,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:55,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:04:58,508 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:04:58,509 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:05:06,937 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:05:06,937 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:05:06,938 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:05:07,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:05:08,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:09,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:11,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:13,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:13,642 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:05:13,642 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:05:15,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:18,472 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:05:27,046 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:05:27,046 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:05:27,047 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:05:27,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:05:28,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:28,868 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:05:28,868 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:05:29,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:31,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:33,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:35,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:44,030 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:05:44,030 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:05:47,156 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:05:47,156 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:05:47,157 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:05:47,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:05:48,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:48,546 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:05:49,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:51,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:53,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:55,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:05:59,162 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:05:59,162 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:06:07,232 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:06:07,233 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:06:07,233 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:06:07,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:06:08,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:09,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:11,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:13,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:14,296 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:06:14,297 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:06:15,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:18,623 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:06:27,319 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:06:27,319 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:06:27,319 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:06:27,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:06:28,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:29,484 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:06:29,484 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:06:29,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:31,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:33,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:35,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:44,647 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:06:44,647 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:06:47,405 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:06:47,405 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:06:47,406 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:06:47,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:06:48,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:48,696 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:06:49,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:51,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:53,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:55,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:06:59,779 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:06:59,780 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:07:07,515 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:07:07,515 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:07:07,516 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:07:07,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:07:08,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:09,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:11,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:13,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:14,909 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:07:14,909 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:07:15,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:18,769 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:07:27,577 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:07:27,577 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:07:27,578 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:07:28,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:28,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:07:29,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:30,090 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:07:30,090 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:07:31,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:33,532 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:35,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:45,300 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:07:45,300 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:07:47,645 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:07:47,646 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:07:47,646 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:07:48,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:48,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:07:48,843 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:07:50,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:52,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:54,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:07:56,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:00,431 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:08:00,432 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:08:07,731 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:08:07,732 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:08:07,732 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:08:08,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:08,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:08:10,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:12,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:14,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:15,564 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:08:15,565 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:08:16,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:18,914 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:08:27,843 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:08:27,844 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:08:27,844 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:08:28,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:08:29,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:30,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:30,717 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:08:30,717 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:08:32,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:34,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:36,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:45,862 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:08:45,863 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:08:47,966 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:08:47,966 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:08:47,967 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:08:48,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:08:48,988 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:08:49,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:50,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:52,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:54,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:08:56,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:00,992 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:09:00,993 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:09:08,062 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:09:08,062 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:09:08,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:09:08,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:09:09,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:10,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:12,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:14,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:16,124 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:09:16,124 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:09:16,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:19,065 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:09:28,182 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:09:28,182 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:09:28,183 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:09:28,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:09:29,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:30,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:31,256 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:09:31,257 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:09:32,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:34,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:36,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:46,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:09:46,433 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:09:48,291 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:09:48,291 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:09:48,292 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:09:48,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:09:49,141 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:09:49,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:50,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:52,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:54,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:09:56,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:01,564 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:10:01,564 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:10:08,383 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:10:08,383 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:10:08,384 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:10:08,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:10:09,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:10,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:12,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:14,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:16,601 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:16,694 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:10:16,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:10:19,216 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:10:28,504 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:10:28,505 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:10:28,505 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:10:28,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:10:29,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:30,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:31,838 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:10:31,838 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:10:32,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:34,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:36,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:47,009 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:10:47,009 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:10:48,557 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:10:48,557 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:10:48,558 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:10:48,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:10:49,289 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:10:49,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:50,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:52,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:54,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:10:56,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:02,141 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:11:02,141 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:11:08,634 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:11:08,635 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:11:08,635 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:11:09,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:09,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:11:10,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:12,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:14,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:16,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:17,271 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:11:17,271 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:11:19,365 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:11:28,697 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:11:28,698 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:11:28,698 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:11:29,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:29,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:11:30,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:32,424 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:11:32,424 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:11:33,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:35,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:37,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:47,633 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:11:47,634 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:11:48,741 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:11:48,742 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:11:48,742 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:11:49,439 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:11:49,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:49,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:11:51,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:53,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:55,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:11:57,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:02,763 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:12:02,763 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:12:08,808 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:12:08,808 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:12:08,809 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:12:09,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:12:10,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:11,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:13,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:15,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:17,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:17,894 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:12:17,894 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:12:19,516 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:12:28,883 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:12:28,883 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:12:28,884 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:12:29,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:12:30,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:31,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:33,051 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:12:33,051 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:12:33,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:35,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:37,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:48,183 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:12:48,183 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:12:48,965 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:12:48,966 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:12:48,966 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:12:49,593 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:12:49,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:12:50,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:51,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:53,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:55,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:12:57,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:03,313 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:13:03,313 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:13:09,036 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:13:09,036 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:13:09,037 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:13:09,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:13:10,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:11,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:13,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:15,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:17,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:18,447 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:13:18,447 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:13:19,671 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:13:29,128 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:13:29,129 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:13:29,129 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:13:29,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:13:30,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:31,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:33,580 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:13:33,580 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:13:33,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:35,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:37,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:48,714 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:13:48,715 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:13:49,201 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:13:49,201 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:13:49,202 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:13:49,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:13:49,747 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:13:50,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:51,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:53,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:55,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:13:57,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:03,853 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:14:03,854 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:14:09,311 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:14:09,311 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:14:09,312 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:14:09,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:14:10,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:11,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:13,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:15,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:17,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:18,990 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:14:18,990 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:14:19,825 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:14:29,354 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:14:29,355 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:14:29,355 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:14:29,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:29,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:14:30,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:32,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:34,122 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:14:34,122 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:14:34,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:36,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:38,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:49,287 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:14:49,287 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:14:49,438 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:14:49,439 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:14:49,439 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:14:49,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:14:49,897 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:14:50,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:52,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:54,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:56,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:14:58,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:04,419 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:15:04,420 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:15:09,509 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:15:09,509 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:15:09,510 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:15:09,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:15:10,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:12,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:14,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:16,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:18,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:19,551 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:15:19,551 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:15:19,973 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:15:29,592 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:15:29,592 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:15:29,593 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:15:29,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:15:30,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:32,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:34,682 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:15:34,682 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:15:34,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:36,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:38,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:49,682 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:15:49,683 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:15:49,683 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:15:49,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:15:49,836 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:15:49,837 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:15:50,046 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:15:50,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:52,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:54,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:56,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:15:58,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:04,970 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:16:04,971 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:16:09,743 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:16:09,743 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:16:09,744 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:16:09,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:16:10,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:11,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:12,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:14,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:16,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:18,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:20,104 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:16:20,104 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:16:20,246 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:16:29,900 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:16:29,900 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:16:29,901 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:16:30,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:30,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:16:31,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:32,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:34,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:35,256 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:16:35,256 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:16:36,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:38,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:49,958 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:16:49,959 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:16:49,959 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:16:50,194 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:16:50,391 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:16:50,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:16:50,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:50,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:16:51,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:52,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:54,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:56,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:16:58,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:05,571 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:17:05,571 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:17:10,020 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:17:10,020 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:17:10,021 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:17:10,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:10,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:17:11,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:12,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:14,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:17,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:19,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:20,269 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:17:20,702 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:17:20,702 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:17:30,100 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:17:30,101 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:17:30,104 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:17:30,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:17:31,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:33,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:35,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:35,856 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:17:35,856 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:17:37,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:39,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:50,174 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:17:50,174 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:17:50,175 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:17:50,345 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:17:50,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:17:51,047 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:17:51,047 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:17:51,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:53,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:55,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:57,799 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:17:59,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:06,212 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:18:06,213 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:18:10,251 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:18:10,251 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:18:10,254 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:18:10,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:18:11,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:13,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:15,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:17,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:19,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:20,414 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:18:21,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:18:21,353 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:18:30,340 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:18:30,340 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:18:30,341 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:18:30,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:18:31,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:33,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:35,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:36,486 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:18:36,487 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:18:37,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:39,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:50,386 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:18:50,387 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:18:50,393 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:18:50,480 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:18:50,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:18:51,663 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:18:51,664 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:18:51,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:53,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:55,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:57,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:18:59,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:06,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:19:06,813 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:19:10,474 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:19:10,474 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:19:10,475 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:19:10,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:19:11,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:13,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:15,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:17,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:19,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:20,550 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:19:21,947 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:19:21,947 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:19:30,538 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:19:30,539 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:19:30,539 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:19:30,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:19:31,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:33,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:35,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:37,110 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:19:37,110 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:19:37,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:39,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:50,621 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:19:50,624 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:19:50,625 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:19:50,625 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:19:50,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:19:51,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:52,301 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:19:52,301 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:19:53,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:55,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:57,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:19:59,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:07,436 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:20:07,436 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:20:10,668 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:20:10,668 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:20:10,669 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:20:10,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:20:11,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:14,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:16,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:18,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:20,695 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:20:20,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:22,569 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:20:22,569 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:20:31,059 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:20:31,059 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:20:31,060 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:20:31,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:31,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:20:32,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:34,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:36,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:37,697 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:20:37,698 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:20:38,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:40,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:50,766 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:20:51,125 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:20:51,125 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:20:51,125 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:20:51,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:51,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:20:52,861 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:20:52,861 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:20:52,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:54,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:56,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:20:58,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:00,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:08,017 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:21:08,017 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:21:11,201 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:21:11,201 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:21:11,202 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:21:11,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:21:12,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:14,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:16,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:18,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:20,837 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:21:23,150 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:21:23,151 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:21:31,280 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:21:31,281 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:21:31,281 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:21:31,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:21:32,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:34,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:36,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:38,283 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:21:38,284 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:21:38,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:40,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:50,909 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:21:51,339 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:21:51,339 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:21:51,340 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:21:51,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:21:52,900 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:53,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:21:53,460 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:21:54,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:56,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:21:58,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:00,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:08,593 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:22:08,594 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:22:11,433 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:22:11,433 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:22:11,434 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:22:11,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:22:12,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:14,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:16,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:18,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:20,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:20,981 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:22:23,725 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:22:23,726 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:22:31,509 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:22:31,509 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:22:31,510 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:22:31,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:22:32,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:34,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:36,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:38,865 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:22:38,865 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:22:38,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:51,054 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:22:51,606 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:22:51,606 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:22:51,607 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:22:51,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:22:52,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:54,034 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:22:54,035 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:22:54,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:56,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:22:58,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:09,167 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:23:09,167 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:23:11,721 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:23:11,721 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:23:11,722 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:23:11,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:23:12,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:14,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:16,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:18,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:21,131 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:23:24,307 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:23:24,307 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:23:31,775 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:23:31,775 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:23:31,776 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:23:31,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:23:32,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:35,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:37,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:39,439 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:23:39,439 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:23:39,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:51,204 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:23:51,918 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:23:51,918 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:23:51,919 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:23:51,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:23:52,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:53,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:54,591 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:23:54,591 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:23:55,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:57,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:23:59,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:09,778 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:24:09,778 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:24:11,998 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:24:11,999 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:24:11,999 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:24:12,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:12,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:24:13,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:15,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:17,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:19,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:21,278 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:24:21,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:24,913 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:24:24,913 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:24:32,061 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:24:32,061 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:24:32,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:24:32,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:32,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:24:33,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:35,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:37,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:39,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:40,043 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:24:40,043 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:24:41,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:51,354 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:24:52,120 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:24:52,120 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:24:52,121 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:24:52,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:52,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:24:53,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:55,185 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:24:55,185 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:24:55,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:57,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:24:59,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:01,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:10,451 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:25:10,452 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:25:12,710 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:25:12,710 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:25:12,711 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:25:12,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:12,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:25:13,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:15,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:17,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:19,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:21,435 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:25:21,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:27,754 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:25:27,754 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:25:32,268 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:25:32,269 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:25:32,269 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:25:33,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:33,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:25:34,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:36,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:38,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:40,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:42,888 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:25:42,888 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:25:51,511 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:25:52,323 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:25:52,323 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:25:52,323 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:25:53,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:53,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:25:54,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:56,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:58,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:25:58,020 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:25:58,021 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:26:00,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:12,418 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:26:12,418 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:26:12,419 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:26:13,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:26:13,159 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:26:13,159 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:26:14,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:16,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:18,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:20,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:21,588 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:26:22,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:28,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:26:28,354 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:26:32,488 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:26:32,489 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:26:32,489 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:26:33,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:26:34,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:36,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:38,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:40,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:43,484 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:26:43,484 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:26:51,666 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:26:52,533 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:26:52,533 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:26:52,534 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:26:53,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:26:54,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:56,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:58,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:26:58,619 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:26:58,619 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:27:00,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:12,619 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:27:12,620 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:27:12,620 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:27:13,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:27:13,772 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:27:13,772 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:27:14,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:16,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:18,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:20,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:21,744 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:27:28,910 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:27:28,910 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:27:32,677 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:27:32,678 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:27:32,678 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:27:33,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:27:34,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:35,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:37,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:39,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:41,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:44,054 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:27:44,055 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:27:51,821 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:27:52,728 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:27:52,728 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:27:52,729 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:27:53,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:27:54,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:55,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:57,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:59,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:27:59,194 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:27:59,195 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:28:01,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:12,832 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:28:12,832 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:28:12,833 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:28:13,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:28:14,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:14,327 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:28:14,328 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:28:15,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:17,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:19,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:21,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:21,896 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:28:29,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:28:29,460 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:28:32,883 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:28:32,884 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:28:32,884 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:28:33,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:28:34,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:35,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:37,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:39,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:41,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:44,590 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:28:44,590 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:28:51,972 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:28:52,968 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:28:52,968 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:28:52,969 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:28:53,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:28:54,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:55,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:57,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:59,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:28:59,722 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:28:59,722 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:29:01,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:13,016 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:29:13,017 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:29:13,017 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:29:13,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:29:14,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:14,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:29:14,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:29:15,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:17,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:19,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:21,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:22,050 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:29:30,149 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:29:30,150 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:29:33,090 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:29:33,090 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:29:33,091 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:29:33,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:29:34,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:35,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:37,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:39,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:41,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:45,281 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:29:45,281 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:29:52,122 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:29:53,213 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:29:53,213 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:29:53,213 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:29:54,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:54,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:29:55,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:57,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:29:59,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:00,414 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:30:00,414 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:30:01,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:13,359 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:30:13,360 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:30:13,360 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:30:14,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:14,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:30:15,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:15,549 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:30:15,555 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:30:17,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:19,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:21,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:22,200 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:30:30,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:30:30,696 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:30:33,396 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:30:33,396 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:30:33,397 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:30:34,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:34,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:30:35,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:37,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:39,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:41,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:45,825 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:30:45,825 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:30:52,276 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:30:53,513 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:30:53,514 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:30:53,514 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:30:54,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:30:55,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:57,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:30:59,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:00,959 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:31:00,960 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:31:01,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:13,608 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:31:13,608 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:31:13,608 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:31:14,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:31:15,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:16,100 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:31:16,100 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:31:18,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:20,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:22,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:22,355 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:31:31,267 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:31:31,268 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:31:33,677 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:31:33,677 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:31:33,678 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:31:34,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:31:35,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:36,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:38,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:40,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:42,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:46,406 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:31:46,407 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:31:52,434 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:31:53,746 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:31:53,746 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:31:53,747 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:31:54,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:31:55,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:56,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:31:58,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:00,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:01,547 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:32:01,548 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:32:02,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:13,811 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:32:13,812 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:32:13,812 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:32:14,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:32:15,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:16,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:16,730 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:32:16,730 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:32:18,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:20,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:22,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:22,511 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:32:31,879 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:32:31,879 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:32:33,905 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:32:33,905 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:32:33,906 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:32:34,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:32:35,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:36,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:38,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:40,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:42,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:47,013 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:32:47,013 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:32:52,588 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:32:53,975 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:32:53,975 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:32:53,976 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:32:54,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:32:55,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:56,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:32:58,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:00,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:02,145 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:33:02,145 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:33:02,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:14,046 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:33:14,047 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:33:14,047 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:33:14,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:33:15,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:16,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:17,285 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:33:17,286 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:33:18,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:20,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:22,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:22,662 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:33:32,450 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:33:32,450 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:33:34,117 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:33:34,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:33:34,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:33:34,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:33:35,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:36,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:38,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:40,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:42,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:47,581 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:33:47,581 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:33:52,734 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:33:54,200 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:33:54,200 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:33:54,201 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:33:54,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:33:55,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:56,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:33:58,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:00,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:02,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:02,715 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:34:02,716 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:34:14,254 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:34:14,255 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:34:14,255 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:34:15,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:15,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:34:16,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:17,888 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:34:17,889 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:34:18,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:20,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:22,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:22,812 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:34:33,068 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:34:33,069 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:34:34,335 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:34:34,335 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:34:34,336 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:34:35,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:34:36,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:38,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:40,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:42,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:48,204 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:34:48,204 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:34:52,887 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:34:54,407 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:34:54,407 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:34:54,407 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:34:55,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:34:56,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:57,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:34:59,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:01,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:03,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:03,336 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:35:03,336 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:35:14,506 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:35:14,506 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:35:14,507 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:35:15,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:35:16,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:17,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:18,475 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:35:18,476 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:35:19,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:21,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:22,961 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:35:23,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:33,660 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:35:33,660 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:35:34,585 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:35:34,586 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:35:34,586 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:35:35,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:35:36,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:37,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:39,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:41,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:43,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:48,792 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:35:48,792 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:35:53,031 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:35:54,679 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:35:54,680 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:35:54,680 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:35:55,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:35:56,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:57,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:35:59,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:01,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:03,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:03,922 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:36:03,922 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:36:14,826 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:36:14,826 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:36:14,829 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:36:15,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:36:16,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:17,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:19,097 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:36:19,098 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:36:19,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:21,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:23,101 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:36:23,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:34,238 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:36:34,239 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:36:34,962 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:36:34,962 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:36:34,968 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:36:35,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:36:36,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:37,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:39,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:41,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:43,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:49,372 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:36:49,372 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:36:53,169 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:36:55,312 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:36:55,312 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:36:55,319 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:36:56,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:56,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:36:57,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:36:59,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:01,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:03,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:04,504 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:37:04,504 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:37:05,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:07,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:09,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:11,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:13,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:15,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:17,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:19,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:19,646 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:37:19,647 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:37:21,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:23,242 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:37:23,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:25,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:27,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:29,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:31,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:33,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:34,777 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:37:34,778 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:37:35,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:37,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:39,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:41,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:43,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:46,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:48,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:49,908 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:37:49,909 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:37:50,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:52,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:53,315 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:37:54,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:56,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:37:59,266 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:37:59,266 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:37:59,266 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:37:59,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:38:05,304 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:38:05,305 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:38:06,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:38:09,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:38:11,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:38:20,452 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:38:20,531 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:38:23,392 DEBUG SenderThread:799749 [sender.py:send():179] send: stats