diff --git "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" --- "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" +++ "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" @@ -15909,3 +15909,3686 @@ 2021-07-17 05:38:20,452 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status 2021-07-17 05:38:20,531 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status 2021-07-17 05:38:23,392 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:38:35,673 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:38:35,673 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:38:50,804 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:38:50,804 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:38:53,464 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:39:05,934 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:39:05,934 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:39:21,065 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:39:21,065 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:39:23,538 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:39:26,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:28,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:30,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:32,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:34,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:36,196 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:39:36,196 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:39:44,555 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:39:44,555 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:39:44,556 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:39:45,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:39:46,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:48,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:50,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:51,325 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:39:51,326 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:39:52,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:39:53,611 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:40:04,610 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:40:04,610 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:40:04,611 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:40:05,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:40:06,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:06,498 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:40:06,498 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:40:08,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:10,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:12,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:21,673 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:40:21,673 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:40:23,686 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:40:24,669 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:40:24,669 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:40:24,669 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:40:25,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:40:26,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:28,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:30,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:32,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:36,802 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:40:36,802 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:40:44,730 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:40:44,730 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:40:44,731 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:40:45,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:40:46,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:48,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:50,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:51,934 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:40:51,934 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:40:52,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:40:53,757 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:41:04,821 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:41:04,822 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:41:04,822 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:41:05,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:41:06,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:07,071 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:41:07,072 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:41:08,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:10,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:12,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:22,232 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:41:22,233 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:41:23,832 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:41:24,920 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:41:24,920 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:41:24,921 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:41:25,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:41:26,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:28,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:30,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:32,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:37,364 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:41:37,365 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:41:44,967 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:41:44,968 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:41:44,968 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:41:45,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:41:47,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:49,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:51,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:52,512 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:41:52,513 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:41:53,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:41:53,907 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:42:05,048 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:42:05,048 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:42:05,049 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:42:05,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:42:07,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:07,645 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:42:07,646 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:42:09,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:11,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:13,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:22,780 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:42:22,780 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:42:23,984 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:42:25,158 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:42:25,159 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:42:25,159 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:42:25,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:42:27,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:29,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:31,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:33,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:37,916 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:42:37,916 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:42:45,209 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:42:45,210 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:42:45,210 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:42:45,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:42:47,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:49,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:51,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:53,046 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:42:53,047 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:42:53,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:42:54,063 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:43:05,293 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:43:05,293 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:43:05,294 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:43:05,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:43:07,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:08,207 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:43:08,207 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:43:09,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:11,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:13,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:23,391 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:43:23,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:43:24,136 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:43:25,392 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:43:25,392 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:43:25,393 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:43:25,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:43:27,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:29,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:31,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:33,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:38,523 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:43:38,523 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:43:45,483 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:43:45,483 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:43:45,483 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:43:46,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:43:47,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:49,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:51,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:53,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:43:53,655 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:43:53,656 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:43:54,209 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:44:05,581 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:44:05,581 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:44:05,582 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:44:06,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:44:07,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:08,848 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:44:08,848 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:44:09,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:11,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:13,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:23,986 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:44:23,987 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:44:24,679 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:44:25,638 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:44:25,639 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:44:25,639 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:44:26,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:44:27,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:29,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:31,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:33,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:39,680 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:44:39,681 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:44:45,769 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:44:45,769 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:44:45,770 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:44:46,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:44:47,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:49,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:51,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:53,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:44:54,360 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:44:54,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:44:54,814 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:45:05,826 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:45:05,826 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:45:05,828 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:45:06,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:45:07,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:09,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:09,999 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:45:09,999 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:45:11,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:13,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:24,435 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:45:25,132 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:45:25,132 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:45:25,923 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:45:25,924 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:45:25,924 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:45:26,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:45:27,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:29,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:31,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:33,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:40,262 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:45:40,262 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:45:45,987 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:45:45,987 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:45:45,989 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:45:46,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:45:47,532 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:49,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:51,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:53,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:45:54,511 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:45:55,395 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:45:55,395 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:46:06,028 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:46:06,029 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:46:06,029 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:46:06,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:46:07,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:10,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:10,554 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:46:10,555 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:46:12,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:14,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:24,583 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:46:25,686 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:46:25,687 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:46:26,076 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:46:26,076 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:46:26,077 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:46:26,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:46:28,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:30,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:32,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:34,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:40,817 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:46:40,817 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:46:46,169 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:46:46,170 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:46:46,170 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:46:46,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:46:48,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:50,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:52,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:54,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:46:54,659 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:46:55,949 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:46:55,950 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:47:06,225 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:47:06,225 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:47:06,225 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:47:06,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:47:08,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:10,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:11,081 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:47:11,081 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:47:12,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:14,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:24,734 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:47:26,214 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:47:26,214 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:47:26,344 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:47:26,344 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:47:26,344 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:47:26,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:47:28,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:30,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:32,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:34,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:41,366 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:47:41,366 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:47:46,364 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:47:46,364 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:47:46,364 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:47:46,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:47:48,586 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:50,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:52,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:54,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:47:54,810 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:47:56,497 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:47:56,497 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:48:06,490 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:48:06,490 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:48:06,491 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:48:06,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:48:08,595 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:10,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:11,630 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:48:11,630 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:48:12,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:14,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:24,885 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:48:26,555 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:48:26,555 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:48:26,556 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:48:26,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:48:26,895 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:48:26,895 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:48:28,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:30,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:32,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:34,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:42,059 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:48:42,059 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:48:46,594 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:48:46,594 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:48:46,595 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:48:46,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:48:48,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:50,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:52,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:54,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:48:54,961 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:48:57,189 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:48:57,189 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:49:06,660 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:49:06,660 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:49:06,661 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:49:07,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:49:08,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:10,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:12,318 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:49:12,318 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:49:12,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:14,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:25,036 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:49:26,707 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:49:26,707 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:49:26,707 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:49:27,481 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:49:27,482 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:49:27,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:49:28,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:30,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:32,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:34,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:42,635 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:49:42,635 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:49:46,791 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:49:46,791 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:49:46,792 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:49:47,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:49:48,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:50,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:52,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:54,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:49:55,112 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:49:57,775 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:49:57,775 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:50:06,891 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:50:06,891 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:50:06,892 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:50:07,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:50:09,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:11,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:12,904 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:50:12,905 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:50:13,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:15,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:25,187 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:50:26,990 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:50:26,990 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:50:26,991 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:50:27,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:50:28,165 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:50:28,165 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:50:29,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:31,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:33,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:35,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:43,328 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:50:43,328 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:50:47,027 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:50:47,027 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:50:47,028 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:50:47,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:50:49,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:51,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:53,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:55,266 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:50:55,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:50:58,461 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:50:58,461 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:51:07,131 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:51:07,132 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:51:07,132 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:51:07,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:51:09,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:11,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:13,593 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:51:13,594 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:51:13,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:15,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:25,340 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:51:27,232 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:51:27,232 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:51:27,233 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:51:27,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:51:28,779 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:51:28,779 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:51:29,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:31,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:33,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:35,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:43,935 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:51:43,936 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:51:47,363 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:51:47,364 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:51:47,364 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:51:47,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:51:49,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:51,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:53,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:55,413 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:51:55,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:51:59,068 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:51:59,069 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:52:07,459 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:52:07,460 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:52:07,460 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:52:07,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:52:09,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:11,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:13,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:14,204 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:52:14,204 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:52:15,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:25,487 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:52:27,519 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:52:27,519 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:52:27,520 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:52:27,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:52:29,377 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:52:29,378 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:52:29,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:31,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:33,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:35,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:44,560 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:52:44,560 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:52:47,578 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:52:47,578 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:52:47,578 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:52:47,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:52:49,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:51,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:53,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:55,561 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:52:55,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:52:59,689 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:52:59,689 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:53:07,642 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:53:07,642 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:53:07,643 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:53:07,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:53:09,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:11,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:13,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:14,821 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:53:14,821 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:53:15,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:25,638 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:53:27,704 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:53:27,704 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:53:27,704 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:53:27,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:53:29,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:29,976 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:53:29,977 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:53:31,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:33,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:35,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:45,148 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:53:45,149 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:53:47,780 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:53:47,781 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:53:47,781 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:53:48,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:53:49,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:51,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:53,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:53:55,709 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:53:55,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:00,278 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:54:00,279 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:54:07,833 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:54:07,834 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:54:07,834 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:54:08,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:54:09,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:11,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:13,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:15,411 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:54:15,411 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:54:15,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:25,784 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:54:27,887 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:54:27,887 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:54:27,888 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:54:28,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:54:29,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:30,622 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:54:30,622 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:54:32,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:34,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:36,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:45,771 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:54:45,771 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:54:47,941 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:54:47,941 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:54:47,941 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:54:48,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:54:49,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:50,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:52,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:54,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:54:55,858 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:54:56,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:00,908 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:55:00,909 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:55:08,029 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:55:08,029 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:55:08,029 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:55:08,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:55:09,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:10,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:12,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:14,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:16,039 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:55:16,039 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:55:16,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:25,935 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:55:28,123 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:55:28,123 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:55:28,124 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:55:28,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:55:30,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:31,194 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:55:31,194 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:55:32,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:34,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:36,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:46,325 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:55:46,326 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:55:48,177 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:55:48,177 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:55:48,178 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:55:48,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:55:50,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:52,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:54,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:55:56,013 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:55:56,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:01,454 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:56:01,454 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:56:08,296 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:56:08,296 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:56:08,297 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:56:08,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:56:10,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:12,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:14,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:16,583 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:56:16,583 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:56:16,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:26,087 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:56:28,350 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:56:28,351 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:56:28,351 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:56:28,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:56:30,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:31,720 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:56:31,721 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:56:32,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:34,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:36,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:46,872 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:56:46,872 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:56:48,425 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:56:48,425 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:56:48,426 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:56:48,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:56:50,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:52,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:54,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:56:56,159 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:56:56,835 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:02,004 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:57:02,005 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:57:08,524 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:57:08,524 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:57:08,524 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:57:08,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:57:10,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:12,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:14,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:16,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:17,134 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:57:17,135 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:57:26,235 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:57:28,631 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:57:28,632 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:57:28,632 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:57:28,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:57:30,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:32,287 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:57:32,287 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:57:32,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:34,852 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:36,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:47,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:57:47,422 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:57:48,688 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:57:48,688 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:57:48,689 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:57:48,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:57:50,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:52,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:54,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:57:56,314 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:57:56,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:02,551 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:58:02,551 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:58:08,734 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:58:08,734 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:58:08,734 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:58:08,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:58:10,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:12,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:14,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:16,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:17,683 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:58:17,683 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:58:26,387 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:58:28,802 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:58:28,802 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:58:28,803 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:58:28,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:58:30,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:32,834 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:58:32,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:58:32,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:34,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:36,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:47,975 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:58:47,976 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:58:48,839 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:58:48,840 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:58:48,840 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:58:48,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:58:50,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:51,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:53,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:55,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:58:56,461 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:58:57,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:03,109 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:59:03,109 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:59:08,878 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:59:08,879 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:59:08,879 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:59:08,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:59:10,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:11,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:13,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:15,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:17,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:18,242 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:59:18,243 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:59:26,528 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:59:28,917 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:59:28,918 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:59:28,918 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:59:29,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:59:30,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:31,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:33,409 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:59:33,409 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:59:33,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:35,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:37,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:48,543 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 05:59:48,544 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 05:59:49,003 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 05:59:49,004 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 05:59:49,004 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 05:59:49,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:49,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 05:59:51,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:53,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:55,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 05:59:56,594 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 05:59:57,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:03,676 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:00:03,676 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:00:09,065 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:00:09,065 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:00:09,066 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:00:09,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:09,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:00:11,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:13,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:15,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:17,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:18,815 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:00:18,815 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:00:26,665 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:00:29,104 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:00:29,105 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:00:29,105 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:00:29,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:00:31,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:33,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:33,947 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:00:33,947 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:00:35,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:37,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:49,082 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:00:49,082 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:00:49,217 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:00:49,217 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:00:49,218 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:00:49,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:00:51,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:53,942 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:55,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:00:56,732 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:00:57,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:04,213 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:01:04,213 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:01:09,283 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:01:09,283 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:01:09,284 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:01:09,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:09,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:01:11,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:13,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:15,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:17,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:19,345 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:01:19,345 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:01:26,799 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:01:29,367 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:01:29,368 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:01:29,369 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:01:29,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:29,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:01:31,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:34,478 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:01:34,478 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:01:34,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:36,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:38,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:49,447 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:01:49,448 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:01:49,448 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:01:49,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:01:49,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:01:49,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:01:50,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:51,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:52,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:54,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:56,867 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:01:56,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:01:58,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:04,815 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:02:04,816 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:02:09,516 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:02:09,516 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:02:09,517 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:02:09,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:02:10,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:11,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:12,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:14,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:16,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:18,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:19,947 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:02:19,947 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:02:26,936 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:02:29,645 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:02:29,645 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:02:29,646 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:02:29,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:02:30,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:31,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:32,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:34,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:35,078 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:02:35,079 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:02:36,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:38,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:49,708 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:02:49,708 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:02:49,709 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:02:49,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:02:50,209 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:02:50,210 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:02:50,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:51,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:52,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:54,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:56,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:02:57,009 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:02:58,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:05,344 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:03:05,345 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:03:09,799 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:03:09,800 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:03:09,800 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:03:10,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:03:11,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:12,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:13,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:15,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:17,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:19,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:20,477 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:03:20,477 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:03:27,080 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:03:29,884 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:03:29,884 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:03:29,885 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:03:30,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:03:31,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:32,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:33,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:35,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:35,661 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:03:35,661 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:03:37,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:39,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:49,988 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:03:49,989 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:03:49,989 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:03:50,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:03:50,837 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:03:50,837 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:03:51,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:52,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:53,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:55,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:57,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:03:57,152 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:03:59,025 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:06,025 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:04:06,025 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:04:10,045 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:04:10,046 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:04:10,046 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:04:11,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:11,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:04:12,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:13,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:15,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:17,034 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:19,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:21,157 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:04:21,158 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:04:27,226 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:04:30,158 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:04:30,158 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:04:30,159 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:04:31,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:31,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:04:32,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:33,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:35,043 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:36,318 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:04:36,318 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:04:37,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:39,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:50,224 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:04:50,224 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:04:50,225 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:04:51,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:51,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:04:51,460 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:04:51,460 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:04:52,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:53,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:56,053 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:04:57,296 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:04:58,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:00,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:06,614 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:05:06,614 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:05:10,288 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:05:10,288 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:05:10,289 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:05:11,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:05:12,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:13,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:14,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:16,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:18,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:20,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:21,747 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:05:21,748 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:05:27,368 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:05:30,343 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:05:30,344 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:05:30,344 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:05:31,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:05:32,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:33,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:34,071 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:36,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:36,881 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:05:36,881 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:05:38,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:40,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:50,405 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:05:50,406 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:05:50,406 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:05:51,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:05:52,049 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:05:52,049 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:05:52,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:53,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:54,080 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:56,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:05:57,443 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:05:58,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:00,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:07,202 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:06:07,203 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:06:10,454 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:06:10,454 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:06:10,455 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:06:11,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:06:12,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:13,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:14,089 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:16,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:18,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:20,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:22,333 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:06:22,333 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:06:27,516 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:06:30,608 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:06:30,608 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:06:30,609 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:06:31,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:06:32,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:33,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:34,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:36,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:37,465 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:06:37,466 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:06:38,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:40,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:50,665 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:06:50,665 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:06:50,666 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:06:51,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:06:52,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:52,655 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:06:52,655 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:06:53,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:54,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:56,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:06:57,592 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:06:58,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:07,819 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:07:07,820 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:07:10,776 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:07:10,776 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:07:10,777 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:07:11,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:07:12,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:13,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:14,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:16,115 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:18,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:20,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:22,950 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:07:22,950 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:07:27,666 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:07:30,864 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:07:30,864 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:07:30,864 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:07:31,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:07:32,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:33,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:34,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:36,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:38,079 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:07:38,080 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:07:38,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:40,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:50,937 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:07:50,937 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:07:50,938 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:07:51,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:07:52,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:53,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:53,210 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:07:53,211 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:07:54,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:56,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:07:57,742 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:07:58,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:00,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:08,371 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:08:08,371 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:08:11,008 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:08:11,009 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:08:11,009 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:08:11,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:08:12,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:13,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:14,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:16,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:18,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:20,147 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:23,502 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:08:23,502 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:08:27,818 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:08:31,058 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:08:31,058 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:08:31,059 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:08:31,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:08:32,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:33,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:34,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:36,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:38,640 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:08:38,640 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:08:39,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:51,145 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:08:51,145 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:08:51,145 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:08:51,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:08:53,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:53,781 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:08:53,782 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:08:55,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:57,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:08:57,893 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:08:59,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:08,942 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:09:08,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:09:11,225 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:09:11,226 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:09:11,226 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:09:12,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:09:13,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:15,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:17,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:19,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:21,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:24,074 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:09:24,074 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:09:27,968 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:09:31,299 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:09:31,299 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:09:31,300 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:09:32,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:09:33,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:35,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:37,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:39,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:39,206 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:09:39,206 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:09:41,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:51,363 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:09:51,364 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:09:51,364 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:09:52,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:09:53,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:54,350 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:09:54,350 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:09:55,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:57,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:09:58,042 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:09:59,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:09,520 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:10:09,520 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:10:11,418 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:10:11,419 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:10:11,419 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:10:12,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:10:13,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:14,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:15,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:17,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:19,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:21,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:24,654 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:10:24,654 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:10:28,116 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:10:31,487 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:10:31,488 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:10:31,488 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:10:32,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:10:33,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:34,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:35,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:37,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:39,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:39,785 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:10:39,785 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:10:51,648 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:10:51,649 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:10:51,649 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:10:52,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:10:53,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:54,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:54,941 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:10:54,941 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:10:55,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:57,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:10:58,191 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:10:59,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:10,098 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:11:10,099 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:11:11,697 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:11:11,698 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:11:11,698 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:11:12,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:11:13,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:14,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:15,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:17,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:19,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:25,232 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:11:25,232 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:11:28,264 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:11:31,829 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:11:31,829 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:11:31,830 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:11:32,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:11:33,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:34,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:35,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:37,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:39,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:40,364 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:11:40,364 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:11:51,897 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:11:51,897 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:11:51,898 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:11:52,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:11:53,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:54,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:55,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:55,495 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:11:55,495 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:11:57,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:11:58,340 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:11:59,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:01,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:10,634 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:12:10,634 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:12:12,012 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:12:12,013 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:12:12,013 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:12:12,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:12:13,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:14,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:15,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:17,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:19,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:21,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:25,767 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:12:25,767 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:12:28,414 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:12:32,123 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:12:32,123 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:12:32,124 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:12:32,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:12:33,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:34,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:35,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:37,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:39,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:40,898 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:12:40,898 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:12:41,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:52,184 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:12:52,185 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:12:52,185 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:12:52,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:12:53,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:54,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:55,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:56,055 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:12:56,056 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:12:57,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:12:58,490 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:12:59,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:01,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:11,196 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:13:11,196 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:13:12,232 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:13:12,232 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:13:12,232 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:13:12,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:13:14,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:16,280 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:18,281 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:20,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:22,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:26,333 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:13:26,334 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:13:28,563 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:13:32,276 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:13:32,277 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:13:32,277 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:13:32,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:13:34,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:36,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:38,290 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:40,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:41,466 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:13:41,466 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:13:42,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:52,347 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:13:52,348 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:13:52,348 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:13:53,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:13:54,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:56,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:56,596 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:13:56,597 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:13:58,298 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:13:58,638 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:14:00,299 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:02,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:11,733 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:14:11,733 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:14:12,386 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:14:12,386 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:14:12,387 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:14:13,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:14:14,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:16,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:18,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:20,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:26,864 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:14:26,865 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:14:28,714 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:14:32,542 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:14:32,542 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:14:32,542 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:14:33,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:14:34,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:36,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:38,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:40,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:41,996 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:14:41,997 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:14:52,597 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:14:52,597 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:14:52,598 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:14:53,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:14:54,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:56,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:57,206 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:14:57,206 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:14:58,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:14:58,790 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:15:00,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:12,340 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:15:12,340 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:15:12,674 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:15:12,674 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:15:12,675 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:15:13,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:15:14,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:16,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:18,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:20,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:22,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:27,470 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:15:27,471 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:15:28,862 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:15:32,709 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:15:32,709 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:15:32,710 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:15:33,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:15:34,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:35,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:36,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:38,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:40,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:42,612 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:15:42,613 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:15:52,745 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:15:52,746 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:15:52,746 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:15:53,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:15:54,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:55,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:56,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:57,743 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:15:57,744 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:15:58,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:15:58,938 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:16:00,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:12,848 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:16:12,848 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:16:12,849 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:16:12,939 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:16:12,939 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:16:13,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:16:14,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:15,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:16,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:18,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:20,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:28,119 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:16:28,119 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:16:29,015 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:16:32,940 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:16:32,941 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:16:32,941 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:16:33,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:16:34,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:35,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:36,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:38,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:40,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:43,251 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:16:43,252 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:16:52,995 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:16:52,995 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:16:52,996 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:16:53,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:16:55,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:57,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:16:58,390 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:16:58,390 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:16:59,093 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:16:59,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:01,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:13,077 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:17:13,077 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:17:13,078 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:17:13,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:17:13,542 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:17:13,542 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:17:15,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:17,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:19,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:21,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:28,715 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:17:28,715 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:17:29,166 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:17:33,124 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:17:33,125 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:17:33,125 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:17:33,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:17:35,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:37,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:39,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:41,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:43,850 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:17:43,851 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:17:53,219 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:17:53,220 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:17:53,220 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:17:53,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:17:55,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:57,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:17:58,984 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:17:58,985 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:17:59,240 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:17:59,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:01,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:13,283 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:18:13,284 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:18:13,284 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:18:13,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:18:14,147 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:18:14,148 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:18:15,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:17,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:19,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:21,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:29,313 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:18:29,314 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:18:29,442 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:18:33,399 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:18:33,399 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:18:33,400 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:18:33,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:18:35,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:37,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:39,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:41,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:44,446 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:18:44,447 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:18:53,466 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:18:53,466 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:18:53,467 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:18:54,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:18:55,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:57,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:59,388 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:18:59,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:18:59,577 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:18:59,577 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:19:01,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:13,574 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:19:13,575 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:19:13,575 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:19:14,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:19:14,820 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:19:14,821 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:19:15,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:17,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:19,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:21,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:29,462 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:19:29,964 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:19:29,964 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:19:33,639 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:19:33,640 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:19:33,642 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:19:34,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:19:35,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:37,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:39,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:41,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:45,096 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:19:45,096 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:19:53,692 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:19:53,692 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:19:53,693 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:19:54,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:19:55,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:56,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:57,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:59,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:19:59,535 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:20:00,247 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:20:00,248 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:20:01,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:13,730 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:20:13,730 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:20:13,733 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:20:14,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:20:15,403 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:20:15,403 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:20:15,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:16,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:17,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:19,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:21,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:29,605 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:20:30,573 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:20:30,573 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:20:33,781 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:20:33,782 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:20:33,782 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:20:34,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:20:35,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:36,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:37,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:39,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:41,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:45,705 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:20:45,705 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:20:53,829 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:20:53,829 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:20:53,832 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:20:54,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:20:55,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:56,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:57,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:59,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:20:59,676 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:21:00,835 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:21:00,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:21:01,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:13,897 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:21:13,898 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:21:13,899 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:21:14,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:21:15,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:15,969 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:21:15,969 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:21:16,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:18,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:20,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:22,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:29,751 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:21:31,190 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:21:31,190 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:21:33,958 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:21:33,958 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:21:33,959 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:21:34,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:21:36,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:38,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:40,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:42,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:46,321 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:21:46,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:21:54,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:21:54,022 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:21:54,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:21:54,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:21:56,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:58,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:21:59,828 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:22:00,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:01,455 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:22:01,455 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:22:02,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:14,110 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:22:14,110 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:22:14,111 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:22:14,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:22:16,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:16,613 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:22:16,613 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:22:18,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:20,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:22,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:29,898 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:22:31,752 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:22:31,753 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:22:34,173 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:22:34,174 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:22:34,174 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:22:34,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:22:36,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:38,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:40,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:42,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:46,883 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:22:46,883 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:22:54,232 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:22:54,233 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:22:54,233 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:22:54,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:22:56,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:58,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:22:59,968 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:23:00,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:02,011 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:23:02,011 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:23:02,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:14,311 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:23:14,312 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:23:14,312 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:23:14,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:23:16,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:17,173 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:23:17,174 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:23:18,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:20,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:22,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:30,037 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:23:32,314 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:23:32,314 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:23:34,389 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:23:34,390 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:23:34,390 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:23:34,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:23:36,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:38,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:40,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:42,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:47,446 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:23:47,446 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:23:54,439 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:23:54,440 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:23:54,440 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:23:54,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:23:56,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:23:58,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:00,110 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:24:00,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:02,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:02,582 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:24:02,582 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:24:14,506 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:24:14,506 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:24:14,507 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:24:14,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:24:16,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:17,741 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:24:17,741 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:24:18,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:20,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:22,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:30,182 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:24:32,898 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:24:32,898 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:24:34,603 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:24:34,603 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:24:34,604 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:24:35,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:24:36,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:38,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:40,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:42,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:48,031 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:24:48,031 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:24:54,698 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:24:54,698 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:24:54,699 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:24:55,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:24:56,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:24:58,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:00,256 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:25:00,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:02,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:03,164 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:25:03,164 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:25:14,775 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:25:14,775 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:25:14,775 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:25:15,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:25:16,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:18,333 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:25:18,333 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:25:18,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:20,586 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:22,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:30,326 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:25:33,485 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:25:33,485 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:25:34,837 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:25:34,837 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:25:34,838 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:25:35,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:25:36,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:37,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:38,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:41,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:43,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:48,615 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:25:48,616 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:25:54,945 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:25:54,946 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:25:54,946 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:25:55,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:25:57,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:25:59,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:00,400 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:26:01,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:03,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:03,753 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:26:03,753 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:26:15,026 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:26:15,027 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:26:15,027 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:26:15,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:26:17,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:18,913 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:26:18,914 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:26:19,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:21,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:23,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:30,471 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:26:34,094 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:26:34,094 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:26:35,075 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:26:35,075 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:26:35,076 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:26:35,620 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:26:37,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:39,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:41,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:43,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:49,223 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:26:49,224 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:26:55,186 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:26:55,187 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:26:55,187 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:26:55,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:26:57,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:26:59,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:00,544 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:27:01,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:03,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:04,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:27:04,353 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:27:15,231 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:27:15,232 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:27:15,232 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:27:15,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:27:17,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:19,489 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:27:19,489 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:27:19,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:21,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:23,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:30,615 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:27:34,625 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:27:34,625 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:27:35,321 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:27:35,321 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:27:35,322 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:27:35,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:27:37,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:39,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:41,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:43,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:49,757 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:27:49,757 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:27:55,388 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:27:55,388 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:27:55,389 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:27:55,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:27:57,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:27:59,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:00,690 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:28:01,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:03,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:04,890 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:28:04,890 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:28:15,450 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:28:15,451 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:28:15,451 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:28:15,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:28:17,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:19,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:20,024 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:28:20,025 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:28:21,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:23,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:30,761 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:28:35,157 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:28:35,158 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:28:35,514 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:28:35,515 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:28:35,515 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:28:35,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:28:37,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:39,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:41,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:43,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:50,289 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:28:50,289 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:28:55,567 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:28:55,567 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:28:55,568 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:28:55,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:28:57,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:28:59,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:00,837 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:29:02,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:04,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:05,419 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:29:05,420 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:29:15,649 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:29:15,650 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:29:15,650 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:29:15,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:29:17,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:18,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:20,549 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:29:20,549 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:29:20,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:22,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:24,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:30,913 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:29:35,685 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:29:35,686 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:29:35,815 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:29:35,815 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:29:35,816 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:29:36,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:29:37,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:38,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:40,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:42,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:44,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:50,818 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:29:50,819 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:29:55,782 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:29:55,782 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:29:55,783 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:29:56,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:29:56,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:29:58,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:00,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:00,986 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:30:02,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:04,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:05,952 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:30:05,953 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:30:15,828 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:30:15,828 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:30:15,828 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:30:16,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:16,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:30:18,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:20,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:21,083 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:30:21,083 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:30:22,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:24,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:31,060 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:30:35,880 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:30:35,881 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:30:35,881 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:30:36,220 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:30:36,221 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:30:36,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:30:38,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:40,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:42,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:44,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:51,357 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:30:51,357 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:30:55,939 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:30:55,940 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:30:55,940 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:30:56,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:30:56,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:30:58,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:00,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:01,137 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:31:02,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:04,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:06,487 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:31:06,488 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:31:15,980 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:31:15,980 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:31:15,981 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:31:16,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:16,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:31:18,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:20,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:21,618 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:31:21,618 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:31:22,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:24,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:31,211 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:31:36,048 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:31:36,049 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:31:36,049 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:31:36,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:36,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:31:36,789 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:31:36,789 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:31:38,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:40,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:42,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:44,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:31:51,926 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:31:51,927 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:31:56,143 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:31:56,143 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:31:56,144 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:31:56,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:31:58,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:00,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:01,285 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:32:02,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:04,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:07,059 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:32:07,059 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:32:16,196 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:32:16,196 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:32:16,197 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:32:16,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:32:18,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:20,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:22,192 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:32:22,193 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:32:22,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:24,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:31,358 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:32:36,278 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:32:36,279 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:32:36,279 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:32:36,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:32:37,374 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:32:37,375 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:32:38,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:40,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:42,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:44,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:52,510 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:32:52,511 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:32:56,329 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:32:56,329 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:32:56,329 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:32:56,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:32:56,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:32:59,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:01,434 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:33:01,799 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:03,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:05,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:07,641 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:33:07,641 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:33:16,429 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:33:16,430 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:33:16,430 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:33:16,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:33:17,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:18,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:19,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:21,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:22,773 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:33:22,773 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:33:23,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:25,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:31,507 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:33:36,468 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:33:36,468 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:33:36,469 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:33:36,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:33:37,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:38,020 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:33:38,021 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:33:38,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:39,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:41,818 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:43,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:45,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:53,196 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:33:53,196 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:33:56,531 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:33:56,531 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:33:56,531 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:33:56,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:33:57,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:33:59,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:01,585 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:34:01,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:03,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:05,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:08,328 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:34:08,328 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:34:16,636 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:34:16,636 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:34:16,637 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:34:16,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:34:17,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:19,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:21,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:23,464 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:34:23,464 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:34:23,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:25,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:31,658 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:34:36,714 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:34:36,715 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:34:36,715 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:34:36,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:34:37,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:38,682 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:34:38,683 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:34:39,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:41,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:43,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:45,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:53,862 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:34:53,862 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:34:56,759 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:34:56,760 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:34:56,760 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:34:56,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:34:57,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:34:59,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:01,726 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:35:01,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:03,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:05,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:08,994 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:35:08,995 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:35:16,794 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:35:16,794 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:35:16,795 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:35:16,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:35:17,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:19,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:21,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:23,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:24,127 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:35:24,127 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:35:25,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:31,793 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:35:36,872 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:35:36,873 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:35:36,873 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:35:36,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:35:37,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:39,264 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:35:39,264 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:35:39,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:41,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:43,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:45,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:54,440 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:35:54,440 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:35:56,949 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:35:56,949 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:35:56,950 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:35:57,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:35:57,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:35:59,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:01,862 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:36:01,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:03,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:05,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:09,576 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:36:09,577 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:36:16,988 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:36:16,989 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:36:16,989 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:36:17,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:17,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:36:19,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:21,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:23,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:24,708 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:36:24,708 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:36:25,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:31,933 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:36:37,038 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:36:37,038 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:36:37,039 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:36:37,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:37,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:36:39,845 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:36:39,846 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:36:39,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:41,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:43,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:45,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:36:55,024 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:36:55,024 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:36:57,092 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:36:57,092 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:36:57,093 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:36:57,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:36:58,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:00,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:02,002 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:37:02,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:04,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:06,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:10,154 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:37:10,154 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:37:17,202 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:37:17,203 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:37:17,203 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:37:17,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:37:18,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:20,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:22,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:24,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:25,288 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:37:25,288 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:37:26,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:32,076 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:37:37,249 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:37:37,250 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:37:37,250 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:37:37,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:37:38,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:40,425 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:37:40,425 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:37:40,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:42,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:44,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:46,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:37:55,568 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:37:55,569 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:37:57,346 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:37:57,346 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:37:57,347 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:37:57,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:37:58,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:00,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:02,150 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:38:02,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:04,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:06,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:10,716 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:38:10,717 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:38:17,444 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:38:17,444 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:38:17,444 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:38:17,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:38:18,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:20,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:22,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:24,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:25,847 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:38:25,847 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:38:26,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:32,224 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:38:37,494 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:38:37,494 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:38:37,494 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:38:37,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:38:38,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:40,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:41,032 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:38:41,033 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:38:42,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:44,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:46,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:38:56,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:38:56,168 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:38:57,541 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:38:57,542 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:38:57,542 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:38:57,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:38:58,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:00,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:02,297 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:39:02,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:04,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:06,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:11,299 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:39:11,299 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:39:17,612 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:39:17,613 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:39:17,613 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:39:17,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:39:18,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:20,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:22,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:24,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:26,429 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:39:26,430 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:39:26,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:32,374 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:39:37,716 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:39:37,716 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:39:37,717 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:39:37,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:39:38,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:40,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:41,575 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:39:41,576 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:39:42,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:44,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:46,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:39:56,780 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:39:56,780 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:39:57,752 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:39:57,753 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:39:57,753 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:39:57,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:39:58,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:00,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:02,448 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:40:02,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:04,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:06,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:11,913 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:40:11,913 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:40:17,789 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:40:17,790 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:40:17,790 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:40:18,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:40:19,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:21,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:23,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:25,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:27,043 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:40:27,043 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:40:32,524 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:40:37,841 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:40:37,842 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:40:37,842 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:40:38,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:40:39,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:41,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:42,177 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:40:42,177 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:40:43,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:45,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:47,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:40:57,309 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:40:57,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:40:57,904 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:40:57,904 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:40:57,905 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:40:58,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:40:59,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:01,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:02,596 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:41:03,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:05,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:07,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:12,444 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:41:12,444 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:41:17,985 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:41:17,986 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:41:17,986 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:41:18,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:41:19,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:20,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:22,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:24,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:26,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:27,575 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:41:27,576 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:41:28,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:32,669 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:41:38,020 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:41:38,021 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:41:38,021 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:41:38,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:41:39,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:40,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:42,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:42,706 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:41:42,707 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:41:44,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:46,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:48,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:41:57,838 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:41:57,838 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:41:58,122 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:41:58,123 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:41:58,123 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:41:59,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:42:00,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:02,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:02,742 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:42:04,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:06,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:08,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:12,968 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:42:12,968 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:42:18,189 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:42:18,190 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:42:18,190 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:42:19,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:42:20,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:22,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:24,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:26,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:28,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:28,099 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:42:28,099 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:42:32,815 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:42:38,261 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:42:38,262 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:42:38,262 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:42:39,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:42:40,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:42,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:43,250 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:42:43,251 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:42:44,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:46,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:48,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:42:58,379 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:42:58,380 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:42:58,515 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:42:58,516 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:42:58,517 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:42:59,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:43:00,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:02,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:02,893 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:43:04,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:06,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:13,523 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:43:13,524 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:43:18,451 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:43:18,451 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:43:18,451 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:43:19,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:43:20,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:22,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:24,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:26,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:28,652 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:43:28,653 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:43:32,970 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:43:38,514 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:43:38,514 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:43:38,515 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:43:39,089 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:43:40,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:42,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:43,784 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:43:43,785 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:43:44,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:46,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:43:58,608 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:43:58,608 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:43:58,608 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:43:58,978 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:43:58,979 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:43:59,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:44:00,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:02,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:03,048 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:44:04,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:06,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:08,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:14,154 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:44:14,154 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:44:18,702 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:44:18,703 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:44:18,703 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:44:19,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:44:20,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:22,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:24,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:26,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:28,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:29,284 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:44:29,284 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:44:33,124 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:44:38,787 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:44:38,787 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:44:38,788 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:44:39,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:44:40,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:42,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:44,416 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:44:44,416 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:44:45,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:47,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:44:58,822 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:44:58,822 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:44:58,823 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:44:59,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:44:59,562 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:44:59,562 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:45:00,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:01,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:03,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:03,196 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:45:05,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:07,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:14,705 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:45:14,705 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:45:18,944 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:45:18,945 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:45:18,945 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:45:19,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:45:20,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:21,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:23,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:25,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:27,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:29,836 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:45:29,837 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:45:33,273 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:45:39,003 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:45:39,004 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:45:39,004 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:45:39,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:45:40,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:41,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:43,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:44,968 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:45:44,969 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:45:45,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:47,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:45:59,069 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:45:59,069 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:45:59,070 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:45:59,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:46:00,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:00,211 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:46:00,211 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:46:01,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:03,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:03,346 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:46:05,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:07,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:15,373 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:46:15,374 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:46:19,134 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:46:19,134 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:46:19,135 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:46:19,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:46:20,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:21,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:23,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:25,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:27,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:29,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:30,505 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:46:30,505 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:46:31,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:33,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:33,423 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:46:35,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:37,166 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:39,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:41,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:43,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:45,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:45,649 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:46:45,649 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:46:47,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:49,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:51,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:53,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:55,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:57,176 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:46:59,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:00,784 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:47:00,785 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:47:01,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:03,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:03,498 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:47:05,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:07,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:09,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:11,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:13,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:15,184 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:15,957 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:47:15,958 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:47:17,185 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:19,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:22,983 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:47:22,983 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:47:22,984 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:47:23,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:23,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:47:26,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:28,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:30,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:31,092 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:47:31,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:47:32,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:33,573 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:47:43,054 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:47:43,055 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:47:43,055 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:47:43,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:47:44,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:45,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:46,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:46,265 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:47:46,265 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:47:48,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:50,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:47:52,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:01,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:48:01,422 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:48:03,125 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:48:03,125 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:48:03,126 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:48:03,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:48:03,648 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:48:04,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:05,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:06,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:08,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:10,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:12,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:16,554 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:48:16,555 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:48:23,210 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:48:23,211 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:48:23,211 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:48:23,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:48:25,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:26,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:28,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:30,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:31,684 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:48:31,685 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:48:32,217 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:33,723 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:48:43,289 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:48:43,289 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:48:43,289 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:48:44,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:48:45,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:46,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:46,867 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:48:46,867 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:48:48,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:50,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:48:52,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:02,060 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:49:02,061 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:49:03,328 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:49:03,328 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:49:03,329 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:49:03,792 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:49:04,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:49:05,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:06,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:08,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:10,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:12,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:17,197 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:49:17,197 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:49:23,373 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:49:23,374 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:49:23,374 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:49:24,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:49:25,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:26,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:28,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:30,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:32,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:32,326 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:49:32,327 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:49:33,863 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:49:43,447 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:49:43,448 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:49:43,448 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:49:44,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:44,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:49:45,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:46,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:47,480 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:49:47,480 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:49:48,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:50,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:49:52,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:02,674 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:50:02,675 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:50:03,512 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:50:03,513 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:50:03,514 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:50:03,935 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:50:04,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:04,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:50:05,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:06,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:08,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:10,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:12,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:17,806 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:50:17,806 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:50:23,558 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:50:23,559 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:50:23,559 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:50:24,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:24,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:50:25,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:26,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:29,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:31,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:32,938 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:50:32,939 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:50:33,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:34,002 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:50:43,625 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:50:43,626 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:50:43,626 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:50:44,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:50:45,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:47,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:48,068 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:50:48,069 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:50:49,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:51,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:50:53,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:03,201 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:51:03,202 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:51:03,680 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:51:03,681 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:51:03,681 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:51:04,077 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:51:04,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:51:05,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:06,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:07,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:09,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:11,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:13,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:18,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:51:18,332 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:51:23,730 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:51:23,730 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:51:23,731 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:51:24,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:51:25,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:26,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:27,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:29,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:31,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:33,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:33,464 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:51:33,465 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:51:34,156 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:51:43,843 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:51:43,843 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:51:43,844 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:51:44,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:51:45,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:46,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:47,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:48,602 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:51:48,602 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:51:49,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:51,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:51:53,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:03,804 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:52:03,805 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:52:03,932 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:52:03,933 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:52:03,933 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:52:04,231 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:52:04,308 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:52:05,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:06,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:07,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:09,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:11,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:13,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:19,043 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:52:19,043 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:52:24,046 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:52:24,046 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:52:24,047 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:52:24,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:52:25,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:26,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:27,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:29,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:31,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:33,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:34,176 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:52:34,176 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:52:34,310 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:52:44,130 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:52:44,130 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:52:44,131 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:52:44,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:52:45,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:46,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:47,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:49,312 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:52:49,312 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:52:49,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:51,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:52:53,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:04,200 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:53:04,201 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:53:04,201 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:53:04,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:53:04,382 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:53:04,530 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:53:04,530 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:53:05,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:06,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:07,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:09,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:11,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:13,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:19,667 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:53:19,668 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:53:24,305 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:53:24,306 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:53:24,306 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:53:24,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:53:25,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:26,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:27,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:29,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:31,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:33,345 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:34,457 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:53:34,800 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:53:34,801 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:53:44,356 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:53:44,356 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:53:44,357 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:53:45,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:45,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:53:46,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:47,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:49,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:49,933 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:53:49,933 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:53:51,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:53:53,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:04,438 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:54:04,439 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:54:04,439 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:54:04,529 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:54:05,144 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:54:05,145 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:54:05,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:05,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:54:06,357 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:07,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:09,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:11,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:13,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:20,276 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:54:20,277 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:54:24,479 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:54:24,479 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:54:24,480 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:54:25,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:25,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:54:26,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:27,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:29,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:31,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:33,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:34,594 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:54:35,405 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:54:35,405 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:54:44,552 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:54:44,552 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:54:44,553 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:54:45,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:45,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:54:46,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:47,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:49,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:50,534 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:54:50,534 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:54:51,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:54:53,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:04,590 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:55:04,591 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:55:04,591 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:55:04,664 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:55:05,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:05,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:55:05,745 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:55:05,746 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:55:06,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:07,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:09,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:11,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:13,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:20,878 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:55:20,878 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:55:24,665 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:55:24,665 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:55:24,666 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:55:25,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:25,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:55:26,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:28,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:30,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:32,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:34,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:34,737 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:55:36,007 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:55:36,008 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:55:44,742 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:55:44,743 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:55:44,743 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:55:45,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:55:46,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:48,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:50,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:51,137 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:55:51,137 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:55:52,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:55:54,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:04,779 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:56:04,779 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:56:04,781 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:56:04,810 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:56:05,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:56:06,275 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:56:06,276 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:56:06,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:08,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:10,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:12,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:14,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:21,516 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:56:21,516 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:56:24,817 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:56:24,818 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:56:24,818 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:56:25,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:56:26,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:28,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:30,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:32,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:34,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:34,883 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:56:36,649 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:56:36,650 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:56:44,972 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:56:44,973 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:56:44,973 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:56:45,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:56:46,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:48,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:50,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:51,780 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:56:51,780 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:56:52,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:56:54,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:04,960 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:57:05,062 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:57:05,062 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:57:05,063 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:57:05,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:57:06,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:06,949 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:57:06,949 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:57:08,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:10,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:12,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:14,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:22,092 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:57:22,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:57:25,132 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:57:25,133 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:57:25,133 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:57:25,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:57:26,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:27,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:28,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:30,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:32,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:34,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:35,038 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:57:37,225 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:57:37,225 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:57:45,217 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:57:45,218 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:57:45,218 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:57:45,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:57:46,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:47,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:48,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:50,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:52,354 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:57:52,354 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:57:52,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:57:54,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:05,115 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:58:05,303 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:58:05,303 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:58:05,304 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:58:05,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:58:06,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:07,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:07,497 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:58:07,497 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:58:08,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:10,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:12,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:14,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:22,673 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:58:22,673 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:58:25,351 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:58:25,351 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:58:25,352 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:58:25,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:58:26,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:27,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:28,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:30,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:32,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:35,193 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:58:37,806 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:58:37,807 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:58:45,463 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:58:45,463 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:58:45,464 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:58:45,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:58:46,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:47,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:48,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:50,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:52,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:58:52,935 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:58:52,936 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:59:05,270 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:59:05,542 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:59:05,542 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:59:05,543 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:59:06,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:06,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:59:07,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:08,067 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:59:08,068 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:59:08,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:10,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:13,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:15,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:23,229 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:59:23,229 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:59:25,625 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:59:25,625 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:59:25,626 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:59:26,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:59:27,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:29,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:31,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:33,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:35,348 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 06:59:35,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:38,360 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:59:38,360 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:59:45,668 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 06:59:45,669 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 06:59:45,669 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 06:59:46,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 06:59:47,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:49,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:51,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:53,492 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 06:59:53,492 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 06:59:53,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 06:59:55,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:05,426 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:00:05,833 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:00:05,833 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:00:05,834 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:00:06,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:00:07,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:08,660 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:00:08,661 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:00:09,521 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:11,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:13,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:15,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:23,799 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:00:23,800 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:00:25,922 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:00:25,923 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:00:25,923 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:00:26,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:00:27,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:29,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:31,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:33,532 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:35,505 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:00:35,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:38,930 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:00:38,931 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:00:45,983 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:00:45,984 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:00:45,984 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:00:46,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:00:47,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:49,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:51,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:53,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:00:54,064 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:00:54,064 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:00:55,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:05,580 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:01:06,031 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:01:06,031 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:01:06,032 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:01:06,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:01:07,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:09,195 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:01:09,195 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:01:09,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:11,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:13,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:15,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:24,367 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:01:24,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:01:26,068 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:01:26,069 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:01:26,069 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:01:26,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:01:27,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:29,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:31,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:33,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:35,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:35,654 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:01:39,498 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:01:39,499 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:01:46,106 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:01:46,106 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:01:46,107 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:01:46,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:01:47,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:49,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:51,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:53,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:01:54,628 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:01:54,629 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:02:05,727 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:02:06,155 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:02:06,156 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:02:06,156 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:02:06,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:02:07,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:09,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:09,837 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:02:09,837 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:02:11,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:13,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:25,020 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:02:25,020 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:02:27,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:27,687 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:02:27,688 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:02:27,688 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:02:28,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:02:29,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:31,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:33,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:35,804 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:02:42,696 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:02:42,696 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:02:46,257 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:02:46,257 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:02:46,258 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:02:46,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:02:47,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:49,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:51,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:53,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:55,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:02:57,827 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:02:57,828 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:03:05,878 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:03:06,295 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:03:06,296 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:03:06,296 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:03:06,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:03:07,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:08,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:10,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:12,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:12,961 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:03:12,961 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:03:14,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:26,356 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:03:26,357 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:03:26,357 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:03:26,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:03:28,103 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:03:28,104 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:03:28,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:30,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:32,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:34,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:35,952 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:03:43,257 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:03:43,257 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:03:46,395 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:03:46,396 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:03:46,396 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:03:46,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:03:48,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:50,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:52,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:54,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:03:58,387 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:03:58,388 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:04:06,028 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:04:06,460 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:04:06,460 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:04:06,461 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:04:06,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:04:08,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:10,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:12,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:13,518 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:04:13,518 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:04:14,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:26,530 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:04:26,531 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:04:26,531 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:04:26,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:04:28,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:28,689 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:04:28,689 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:04:30,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:32,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:34,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:36,095 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:04:43,832 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:04:43,832 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:04:46,588 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:04:46,589 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:04:46,589 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:04:46,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:04:48,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:50,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:52,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:54,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:04:58,964 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:04:58,964 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:05:06,166 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:05:06,674 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:05:06,674 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:05:06,674 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:05:07,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:05:08,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:10,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:12,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:14,095 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:05:14,096 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:05:14,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:26,739 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:05:26,739 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:05:26,740 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:05:27,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:05:28,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:29,277 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:05:29,277 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:05:30,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:32,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:34,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:36,235 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:05:44,436 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:05:44,436 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:05:46,787 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:05:46,788 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:05:46,788 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:05:47,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:05:48,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:50,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:52,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:54,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:05:59,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:05:59,567 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:06:06,305 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:06:06,927 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:06:06,928 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:06:06,928 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:06:07,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:06:08,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:10,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:12,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:14,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:14,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:06:14,696 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:06:26,978 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:06:26,979 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:06:26,979 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:06:27,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:06:28,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:29,881 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:06:29,881 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:06:30,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:32,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:34,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:36,379 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:06:45,012 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:06:45,013 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:06:47,034 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:06:47,035 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:06:47,035 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:06:47,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:06:48,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:50,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:52,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:06:54,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:00,149 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:07:00,149 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:07:06,455 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:07:07,079 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:07:07,080 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:07:07,080 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:07:07,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:07:08,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:10,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:13,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:15,282 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:07:15,282 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:07:15,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:27,116 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:07:27,116 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:07:27,117 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:07:27,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:07:28,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:29,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:30,456 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:07:30,456 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:07:31,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:33,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:35,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:36,532 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:07:45,636 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:07:45,637 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:07:47,179 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:07:47,180 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:07:47,180 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:07:47,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:07:48,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:49,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:51,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:53,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:07:55,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:00,765 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:08:00,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:08:06,601 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:08:07,230 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:08:07,231 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:08:07,231 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:08:07,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:08:08,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:09,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:11,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:13,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:15,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:15,894 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:08:15,894 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:08:27,285 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:08:27,285 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:08:27,285 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:08:27,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:08:28,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:29,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:31,033 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:08:31,034 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:08:31,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:33,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:35,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:36,673 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:08:46,169 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:08:46,170 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:08:47,377 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:08:47,377 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:08:47,378 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:08:47,740 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:08:49,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:51,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:53,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:08:55,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:01,301 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:09:01,302 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:09:06,739 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:09:07,467 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:09:07,467 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:09:07,468 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:09:07,749 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:09:09,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:11,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:13,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:15,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:16,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:09:16,433 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:09:27,535 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:09:27,535 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:09:27,536 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:09:27,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:09:29,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:31,562 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:09:31,563 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:09:31,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:33,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:35,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:36,808 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:09:46,694 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:09:46,694 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:09:47,599 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:09:47,600 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:09:47,600 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:09:47,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:09:49,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:51,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:53,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:09:55,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:01,828 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:10:01,828 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:10:06,876 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:10:07,664 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:10:07,665 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:10:07,665 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:10:07,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:10:09,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:11,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:13,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:15,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:16,959 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:10:16,960 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:10:27,702 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:10:27,702 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:10:27,702 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:10:27,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:10:29,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:31,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:32,092 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:10:32,092 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:10:33,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:36,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:36,946 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:10:47,228 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:10:47,228 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:10:47,740 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:10:47,741 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:10:47,741 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:10:47,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:10:48,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:49,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:50,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:52,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:54,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:10:56,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:02,359 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:11:02,360 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:11:07,015 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:11:07,835 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:11:07,836 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:11:07,836 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:11:08,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:08,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:11:09,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:10,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:12,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:14,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:16,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:17,500 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:11:17,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:11:27,931 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:11:27,932 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:11:27,932 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:11:28,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:28,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:11:29,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:30,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:32,633 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:11:32,634 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:11:32,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:34,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:36,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:37,084 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:11:47,764 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:11:47,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:11:47,972 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:11:47,973 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:11:47,973 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:11:48,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:11:49,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:50,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:52,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:54,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:11:56,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:02,895 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:12:02,896 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:12:07,150 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:12:08,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:12:08,023 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:12:08,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:12:08,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:12:09,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:10,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:12,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:14,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:16,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:18,027 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:12:18,027 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:12:28,110 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:12:28,110 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:12:28,110 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:12:28,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:12:29,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:30,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:32,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:33,160 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:12:33,160 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:12:34,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:36,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:37,216 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:12:48,148 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:12:48,148 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:12:48,149 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:12:48,291 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:12:48,292 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:12:48,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:48,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:12:49,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:50,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:52,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:54,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:12:56,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:03,447 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:13:03,448 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:13:07,286 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:13:08,207 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:13:08,207 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:13:08,208 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:13:08,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:13:09,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:10,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:12,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:14,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:16,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:18,579 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:13:18,579 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:13:28,265 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:13:28,266 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:13:28,266 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:13:28,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:13:29,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:30,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:32,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:33,712 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:13:33,712 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:13:34,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:36,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:37,354 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:13:48,315 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:13:48,315 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:13:48,316 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:13:48,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:13:48,879 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:13:48,879 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:13:49,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:50,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:52,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:54,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:13:56,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:04,034 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:14:04,034 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:14:07,424 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:14:08,384 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:14:08,384 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:14:08,385 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:14:08,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:14:09,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:10,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:12,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:14,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:16,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:19,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:14:19,168 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:14:28,485 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:14:28,486 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:14:28,486 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:14:28,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:28,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:14:30,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:32,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:34,299 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:14:34,299 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:14:34,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:36,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:37,495 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:14:48,523 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:14:48,523 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:14:48,524 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:14:48,900 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:14:49,458 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:14:49,459 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:14:50,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:52,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:54,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:14:56,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:04,612 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:15:04,612 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:15:07,567 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:15:08,597 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:15:08,597 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:15:08,597 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:15:08,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:15:10,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:12,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:14,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:16,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:19,742 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:15:19,742 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:15:28,706 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:15:28,707 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:15:28,707 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:15:28,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:15:30,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:33,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:34,872 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:15:34,872 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:15:35,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:37,640 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:15:37,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:48,752 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:15:48,752 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:15:48,752 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:15:48,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:15:50,027 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:15:50,027 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:15:50,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:51,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:53,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:55,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:15:57,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:05,165 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:16:05,165 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:16:07,713 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:16:08,829 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:16:08,830 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:16:08,830 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:16:08,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:16:10,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:11,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:13,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:15,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:17,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:20,295 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:16:20,296 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:16:28,911 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:16:28,911 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:16:28,912 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:16:28,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:16:30,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:31,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:33,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:35,429 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:16:35,429 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:16:35,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:37,785 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:16:37,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:48,955 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:16:48,956 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:16:48,956 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:16:49,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:16:50,561 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:16:50,561 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:16:50,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:51,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:53,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:55,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:16:57,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:05,697 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:17:05,698 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:17:07,858 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:17:09,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:17:09,022 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:17:09,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:17:09,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:17:10,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:11,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:13,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:15,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:17,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:20,834 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:17:20,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:17:29,094 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:17:29,095 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:17:29,095 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:17:29,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:29,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:17:30,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:31,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:33,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:35,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:36,005 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:17:36,006 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:17:37,932 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:17:37,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:49,156 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:17:49,157 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:17:49,157 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:17:49,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:17:50,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:51,160 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:17:51,161 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:17:51,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:53,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:55,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:17:57,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:06,310 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:18:06,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:18:08,008 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:18:09,194 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:18:09,195 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:18:09,195 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:18:09,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:18:10,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:11,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:13,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:15,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:17,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:21,442 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:18:21,442 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:18:29,244 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:18:29,245 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:18:29,245 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:18:30,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:18:32,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:34,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:36,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:36,573 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:18:36,574 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:18:38,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:38,082 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:18:49,320 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:18:49,321 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:18:49,321 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:18:50,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:18:51,721 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:18:51,722 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:18:52,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:54,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:56,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:18:58,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:06,866 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:19:06,866 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:19:08,157 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:19:09,382 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:19:09,382 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:19:09,383 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:19:10,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:10,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:19:12,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:14,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:16,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:18,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:21,999 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:19:22,000 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:19:29,474 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:19:29,474 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:19:29,475 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:19:30,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:19:32,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:34,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:36,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:37,131 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:19:37,131 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:19:38,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:38,234 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:19:49,519 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:19:49,520 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:19:49,520 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:19:50,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:19:52,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:52,280 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:19:52,281 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:19:54,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:56,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:19:58,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:07,412 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:20:07,412 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:20:08,305 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:20:09,589 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:20:09,589 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:20:09,590 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:20:10,052 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:20:12,053 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:14,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:16,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:18,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:22,541 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:20:22,541 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:20:29,657 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:20:29,658 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:20:29,658 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:20:30,062 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:20:32,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:34,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:36,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:37,674 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:20:37,675 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:20:38,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:38,376 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:20:49,758 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:20:49,758 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:20:49,759 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:20:50,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:20:52,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:53,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:20:53,562 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:20:53,562 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:20:58,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:21:00,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:21:02,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:21:08,448 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:21:08,728 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:21:08,729 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status