diff --git "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" --- "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" +++ "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" @@ -3933,3 +3933,3824 @@ 2021-07-16 02:53:03,821 DEBUG SenderThread:761704 [sender.py:send():179] send: stats 2021-07-16 02:53:06,414 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status 2021-07-16 02:53:06,519 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:53:21,651 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:53:21,651 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:53:33,895 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:53:36,783 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:53:36,784 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:53:51,916 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:53:51,917 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:54:03,976 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:54:07,050 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:54:07,051 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:54:22,184 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:54:22,184 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:54:34,049 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:54:37,319 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:54:37,319 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:54:40,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:54:42,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:54:44,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:54:46,263 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:54:48,263 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:54:52,455 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:54:52,455 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:54:59,621 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:54:59,622 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:54:59,622 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:55:00,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:55:01,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:02,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:04,125 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:55:04,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:06,271 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:07,590 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:55:07,590 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:55:08,272 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:19,661 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:55:19,662 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:55:19,662 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:55:20,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:55:21,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:22,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:22,725 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:55:22,725 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:55:24,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:26,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:28,281 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:34,193 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:55:37,871 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:55:37,871 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:55:39,702 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:55:39,702 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:55:39,703 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:55:40,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:55:41,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:42,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:44,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:46,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:48,290 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:55:53,003 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:55:53,003 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:55:59,743 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:55:59,743 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:55:59,744 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:56:00,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:56:01,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:02,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:04,267 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:56:04,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:06,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:08,137 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:56:08,137 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:56:08,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:19,784 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:56:19,784 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:56:19,784 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:56:20,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:56:21,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:22,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:23,266 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:56:23,267 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:56:24,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:26,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:28,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:34,343 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:56:38,399 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:56:38,399 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:56:39,823 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:56:39,823 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:56:39,824 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:56:40,310 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:56:41,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:42,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:44,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:46,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:48,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:56:53,530 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:56:53,530 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:56:59,862 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:56:59,863 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:56:59,864 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:57:00,318 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:57:01,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:02,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:04,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:04,418 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:57:06,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:08,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:08,661 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:57:08,661 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:57:19,901 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:57:19,901 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:57:19,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:57:20,327 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:57:21,327 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:22,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:23,794 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:57:23,794 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:57:24,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:26,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:28,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:34,493 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:57:38,927 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:57:38,927 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:57:39,942 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:57:39,942 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:57:39,943 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:57:40,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:57:41,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:42,336 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:44,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:46,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:48,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:57:54,056 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:57:54,057 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:57:59,982 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:57:59,982 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:57:59,983 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:58:00,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:58:01,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:02,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:04,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:04,569 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:58:06,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:08,347 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:09,187 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:58:09,187 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:58:20,021 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:58:20,021 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:58:20,022 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:58:20,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:58:21,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:22,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:24,320 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:58:24,320 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:58:24,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:26,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:28,355 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:34,641 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:58:39,452 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:58:39,453 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:58:40,059 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:58:40,059 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:58:40,060 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:58:40,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:58:41,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:42,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:44,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:46,362 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:48,363 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:58:54,583 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:58:54,584 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:59:00,097 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:59:00,098 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:59:00,098 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:59:00,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:59:01,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:02,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:04,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:04,706 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:59:06,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:08,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:09,715 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:59:09,715 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:59:20,135 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:59:20,136 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:59:20,136 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:59:20,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:59:21,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:22,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:24,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:24,852 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:59:24,853 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:59:26,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:28,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:34,776 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 02:59:39,983 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:59:39,984 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 02:59:40,185 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 02:59:40,185 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 02:59:40,186 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 02:59:40,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 02:59:42,385 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:44,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:46,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:48,388 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 02:59:55,118 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 02:59:55,118 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:00:00,222 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:00:00,222 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:00:00,222 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:00:00,392 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:00,393 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:00:02,393 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:04,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:04,844 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:00:06,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:08,396 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:10,248 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:00:10,249 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:00:20,262 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:00:20,262 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:00:20,262 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:00:20,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:20,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:00:22,401 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:23,401 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:25,380 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:00:25,380 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:00:25,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:27,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:29,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:34,910 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:00:40,299 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:00:40,299 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:00:40,300 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:00:40,409 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:00:40,536 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:00:40,536 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:00:42,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:43,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:45,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:47,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:49,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:00:55,679 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:00:55,679 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:01:00,335 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:01:00,335 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:01:00,336 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:01:00,418 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:01:02,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:03,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:04,977 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:01:05,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:07,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:09,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:10,810 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:01:10,811 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:01:20,372 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:01:20,372 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:01:20,373 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:01:20,426 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:01:22,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:23,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:25,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:25,941 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:01:25,941 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:01:27,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:29,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:35,047 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:01:40,411 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:01:40,411 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:01:40,411 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:01:40,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:01:41,096 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:01:41,097 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:01:42,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:43,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:45,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:47,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:49,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:01:56,227 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:01:56,227 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:02:00,453 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:02:00,453 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:02:00,454 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:02:01,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:01,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:02:02,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:03,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:05,120 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:02:05,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:07,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:09,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:11,362 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:02:11,362 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:02:20,492 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:02:20,492 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:02:20,493 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:02:21,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:21,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:02:22,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:23,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:25,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:26,491 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:02:26,491 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:02:27,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:29,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:35,193 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:02:40,527 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:02:40,527 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:02:40,528 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:02:41,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:41,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:02:41,657 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:02:41,657 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:02:42,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:43,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:45,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:47,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:49,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:02:56,797 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:02:56,798 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:03:00,564 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:03:00,564 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:03:00,564 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:03:01,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:01,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:03:02,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:03,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:05,262 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:03:05,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:07,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:09,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:11,930 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:03:11,931 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:03:20,601 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:03:20,601 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:03:20,602 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:03:21,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:21,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:03:22,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:23,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:25,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:27,086 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:03:27,087 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:03:27,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:29,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:35,337 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:03:40,636 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:03:40,636 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:03:40,637 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:03:41,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:41,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:03:42,218 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:03:42,219 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:03:42,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:43,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:45,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:47,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:49,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:03:57,374 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:03:57,375 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:04:00,674 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:04:00,675 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:04:00,675 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:04:01,496 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:01,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:04:02,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:03,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:05,411 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:04:05,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:07,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:09,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:12,507 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:04:12,507 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:04:20,710 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:04:20,711 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:04:20,711 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:04:21,504 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:21,504 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:04:22,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:23,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:25,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:27,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:27,638 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:04:27,639 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:04:29,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:35,484 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:04:40,751 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:04:40,751 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:04:40,752 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:04:41,512 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:41,512 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:04:42,512 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:42,776 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:04:42,776 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:04:43,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:45,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:47,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:49,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:04:57,907 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:04:57,908 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:05:00,798 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:05:00,798 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:05:00,799 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:05:01,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:01,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:05:02,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:03,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:05,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:05,555 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:05:07,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:09,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:13,041 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:05:13,042 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:05:20,837 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:05:20,837 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:05:20,838 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:05:21,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:05:22,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:24,529 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:26,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:28,171 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:05:28,171 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:05:28,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:30,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:35,627 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:05:40,877 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:05:40,878 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:05:40,878 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:05:41,537 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:05:42,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:43,330 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:05:43,330 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:05:44,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:46,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:48,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:50,542 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:05:58,470 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:05:58,470 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:06:00,916 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:06:00,917 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:06:00,917 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:06:01,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:06:02,547 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:04,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:05,701 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:06:06,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:08,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:10,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:13,601 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:06:13,602 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:06:20,964 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:06:20,964 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:06:20,965 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:06:21,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:06:22,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:24,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:26,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:28,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:28,733 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:06:28,733 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:06:30,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:35,776 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:06:41,002 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:06:41,002 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:06:41,003 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:06:41,563 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:06:42,564 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:43,869 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:06:43,874 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:06:44,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:46,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:48,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:50,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:06:59,007 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:06:59,008 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:07:01,044 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:07:01,045 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:07:01,045 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:07:01,572 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:07:02,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:04,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:05,852 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:07:06,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:08,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:10,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:14,142 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:07:14,142 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:07:21,091 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:07:21,091 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:07:21,092 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:07:21,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:07:22,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:24,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:26,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:28,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:29,273 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:07:29,274 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:07:30,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:35,927 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:07:41,131 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:07:41,131 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:07:41,132 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:07:41,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:07:42,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:43,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:44,437 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:07:44,437 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:07:44,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:46,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:48,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:50,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:07:59,686 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:07:59,687 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:08:01,173 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:08:01,173 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:08:01,174 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:08:01,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:08:02,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:03,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:04,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:06,002 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:08:06,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:08,603 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:10,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:14,819 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:08:14,820 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:08:21,209 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:08:21,209 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:08:21,210 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:08:21,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:08:22,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:23,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:24,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:26,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:28,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:29,950 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:08:29,950 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:08:30,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:36,075 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:08:41,247 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:08:41,247 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:08:41,247 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:08:41,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:08:42,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:43,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:44,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:45,079 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:08:45,080 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:08:46,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:48,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:08:50,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:00,210 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:09:00,210 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:09:01,285 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:09:01,285 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:09:01,286 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:09:01,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:09:02,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:03,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:04,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:06,148 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:09:06,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:08,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:10,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:15,340 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:09:15,340 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:09:21,325 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:09:21,325 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:09:21,326 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:09:21,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:09:22,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:23,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:24,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:26,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:28,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:30,470 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:09:30,470 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:09:30,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:36,220 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:09:41,370 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:09:41,371 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:09:41,371 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:09:41,644 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:09:42,644 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:43,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:44,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:45,601 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:09:45,601 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:09:46,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:48,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:09:50,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:00,731 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:10:00,732 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:10:01,414 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:10:01,415 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:10:01,415 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:10:01,653 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:10:02,653 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:03,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:04,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:06,294 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:10:06,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:08,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:10,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:15,861 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:10:15,861 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:10:21,451 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:10:21,451 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:10:21,452 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:10:21,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:10:22,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:23,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:24,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:26,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:28,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:30,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:30,993 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:10:30,994 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:10:36,366 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:10:41,487 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:10:41,488 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:10:41,488 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:10:41,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:10:42,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:43,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:44,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:46,129 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:10:46,130 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:10:46,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:48,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:10:50,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:01,260 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:11:01,261 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:11:01,525 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:11:01,525 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:11:01,526 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:11:01,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:11:02,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:03,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:04,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:06,439 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:11:06,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:08,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:10,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:16,391 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:11:16,391 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:11:21,562 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:11:21,562 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:11:21,563 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:11:21,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:11:22,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:23,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:24,690 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:26,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:28,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:30,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:31,520 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:11:31,521 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:11:36,516 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:11:41,600 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:11:41,601 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:11:41,601 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:11:41,698 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:11:42,698 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:43,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:44,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:46,652 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:11:46,653 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:11:46,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:48,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:11:50,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:01,641 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:12:01,641 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:12:01,642 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:12:01,706 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:12:01,817 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:12:01,818 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:12:02,707 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:03,707 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:04,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:06,592 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:12:06,709 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:08,709 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:10,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:17,091 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:12:17,091 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:12:21,683 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:12:21,683 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:12:21,684 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:12:21,716 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:12:22,716 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:23,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:24,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:26,718 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:28,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:30,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:32,222 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:12:32,222 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:12:36,663 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:12:41,721 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:12:41,721 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:12:41,722 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:12:41,724 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:12:42,725 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:43,725 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:44,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:46,727 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:47,355 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:12:47,356 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:12:48,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:12:50,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:01,763 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:13:01,763 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:13:01,764 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:13:02,519 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:13:02,520 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:13:02,734 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:02,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:13:03,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:04,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:06,729 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:13:06,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:08,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:10,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:17,670 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:13:17,670 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:13:21,799 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:13:21,800 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:13:21,800 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:13:22,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:22,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:13:23,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:24,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:26,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:28,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:30,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:32,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:13:32,801 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:13:36,797 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:13:41,858 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:13:41,859 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:13:41,859 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:13:42,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:42,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:13:43,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:44,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:46,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:47,943 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:13:47,944 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:13:48,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:13:50,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:01,894 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:14:01,895 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:14:01,895 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:14:02,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:02,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:14:03,083 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:14:03,084 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:14:03,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:04,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:06,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:06,864 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:14:08,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:10,765 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:18,215 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:14:18,216 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:14:21,937 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:14:21,938 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:14:21,938 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:14:22,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:22,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:14:23,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:24,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:26,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:28,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:30,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:33,357 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:14:33,358 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:14:36,934 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:14:41,976 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:14:41,976 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:14:41,977 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:14:42,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:42,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:14:43,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:44,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:46,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:48,495 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:14:48,496 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:14:48,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:50,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:53,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:55,786 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:57,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:14:59,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:01,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:03,632 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:15:03,632 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:15:03,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:05,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:07,000 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:15:07,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:09,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:11,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:13,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:15,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:17,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:18,766 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:15:18,767 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:15:19,796 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:21,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:23,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:25,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:27,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:29,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:31,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:33,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:33,916 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:15:33,916 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:15:35,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:37,072 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:15:37,803 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:39,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:41,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:43,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:45,829 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:15:45,829 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:15:45,829 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:15:46,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:15:47,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:49,048 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:15:49,049 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:15:49,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:51,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:15:53,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:04,189 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:16:04,189 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:16:05,882 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:16:05,882 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:16:05,882 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:16:06,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:16:07,142 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:16:07,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:08,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:09,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:11,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:13,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:19,322 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:16:19,322 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:16:25,929 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:16:25,929 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:16:25,930 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:16:26,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:16:27,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:28,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:29,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:31,827 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:33,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:34,455 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:16:34,455 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:16:37,213 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:16:45,967 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:16:45,967 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:16:45,968 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:16:46,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:16:47,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:48,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:49,586 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:16:49,586 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:16:49,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:51,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:16:53,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:04,724 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:17:04,724 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:17:06,008 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:17:06,009 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:17:06,009 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:17:06,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:17:07,283 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:17:07,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:08,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:09,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:11,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:13,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:19,855 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:17:19,856 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:17:26,054 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:17:26,054 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:17:26,054 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:17:26,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:17:27,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:28,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:29,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:31,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:33,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:34,988 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:17:34,989 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:17:37,354 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:17:46,092 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:17:46,093 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:17:46,093 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:17:46,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:46,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:17:47,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:49,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:50,118 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:17:50,119 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:17:51,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:17:53,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:05,249 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:18:05,249 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:18:06,141 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:18:06,141 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:18:06,142 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:18:06,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:06,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:18:07,428 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:18:07,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:09,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:11,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:13,873 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:20,381 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:18:20,382 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:18:26,182 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:18:26,182 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:18:26,183 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:18:26,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:26,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:18:27,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:29,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:31,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:33,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:35,517 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:18:35,517 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:18:37,500 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:18:46,224 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:18:46,225 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:18:46,225 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:18:46,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:46,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:18:47,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:49,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:50,648 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:18:50,648 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:18:51,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:18:53,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:05,780 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:19:05,781 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:19:06,265 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:19:06,265 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:19:06,266 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:19:06,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:06,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:19:07,573 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:19:07,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:09,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:11,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:13,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:20,910 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:19:20,910 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:19:26,309 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:19:26,309 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:19:26,310 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:19:26,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:26,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:19:27,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:29,905 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:31,905 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:33,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:36,041 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:19:36,041 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:19:37,645 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:19:46,348 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:19:46,348 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:19:46,349 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:19:46,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:46,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:19:48,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:50,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:51,171 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:19:51,171 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:19:52,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:19:54,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:06,306 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:20:06,307 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:20:06,438 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:20:06,438 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:20:06,439 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:20:06,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:06,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:20:07,718 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:20:08,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:10,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:12,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:14,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:21,444 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:20:21,444 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:20:26,426 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:20:26,426 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:20:26,426 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:20:26,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:20:28,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:30,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:32,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:34,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:36,582 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:20:36,582 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:20:37,790 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:20:46,716 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:20:46,717 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:20:46,717 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:20:46,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:20:48,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:50,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:51,711 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:20:51,711 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:20:52,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:20:54,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:06,757 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:21:06,758 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:21:06,758 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:21:06,866 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:21:06,867 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:21:06,946 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:21:07,863 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:21:08,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:10,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:12,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:14,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:22,000 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:21:22,001 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:21:26,794 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:21:26,795 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:21:26,795 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:21:26,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:21:28,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:30,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:32,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:34,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:37,133 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:21:37,134 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:21:37,936 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:21:46,839 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:21:46,840 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:21:46,840 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:21:46,963 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:21:48,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:49,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:50,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:52,383 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:21:52,383 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:21:52,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:21:54,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:06,882 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:22:06,883 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:22:06,883 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:22:06,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:22:07,529 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:22:07,530 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:22:08,008 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:22:08,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:09,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:10,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:12,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:14,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:22,666 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:22:22,666 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:22:26,921 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:22:26,922 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:22:26,922 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:22:26,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:22:28,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:29,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:30,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:32,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:34,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:37,798 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:22:37,798 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:22:38,080 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:22:46,972 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:22:46,972 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:22:46,972 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:22:46,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:22:48,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:49,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:50,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:52,930 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:22:52,931 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:22:52,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:22:54,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:07,014 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:23:07,014 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:23:07,015 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:23:07,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:23:08,070 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:23:08,071 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:23:08,206 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:23:08,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:09,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:11,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:13,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:15,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:23,212 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:23:23,213 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:23:27,053 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:23:27,054 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:23:27,054 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:23:28,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:23:29,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:30,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:31,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:33,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:35,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:38,226 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:23:38,343 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:23:38,344 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:23:47,094 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:23:47,094 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:23:47,095 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:23:48,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:48,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:23:49,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:51,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:53,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:23:53,473 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:23:53,473 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:23:55,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:07,134 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:24:07,135 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:24:07,135 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:24:08,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:08,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:24:08,296 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:24:08,639 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:24:08,639 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:24:09,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:11,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:13,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:15,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:23,770 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:24:23,770 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:24:27,171 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:24:27,172 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:24:27,172 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:24:28,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:24:29,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:30,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:31,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:33,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:35,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:38,369 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:24:38,904 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:24:38,904 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:24:47,210 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:24:47,210 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:24:47,211 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:24:48,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:24:49,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:50,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:51,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:53,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:24:54,036 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:24:54,036 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:24:55,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:07,247 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:25:07,247 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:25:07,248 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:25:08,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:25:08,440 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:25:09,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:09,167 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:25:09,168 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:25:10,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:11,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:13,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:15,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:24,308 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:25:24,308 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:25:27,286 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:25:27,287 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:25:27,287 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:25:28,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:25:29,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:30,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:31,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:33,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:35,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:38,517 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:25:39,437 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:25:39,437 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:25:47,331 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:25:47,331 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:25:47,332 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:25:48,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:25:49,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:50,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:51,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:53,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:25:54,568 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:25:54,569 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:25:55,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:07,371 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:26:07,372 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:26:07,372 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:26:08,078 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:26:08,596 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:26:09,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:09,708 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:26:09,709 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:26:10,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:11,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:13,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:15,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:24,841 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:26:24,841 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:26:27,414 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:26:27,415 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:26:27,415 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:26:28,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:26:29,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:30,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:31,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:33,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:35,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:38,673 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:26:39,984 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:26:39,984 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:26:47,454 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:26:47,455 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:26:47,455 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:26:48,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:26:49,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:50,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:51,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:53,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:55,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:26:55,114 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:26:55,114 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:27:07,500 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:27:07,501 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:27:07,501 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:27:08,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:27:08,750 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:27:09,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:10,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:10,248 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:27:10,248 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:27:11,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:13,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:15,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:25,404 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:27:25,405 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:27:27,538 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:27:27,538 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:27:27,539 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:27:28,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:27:29,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:30,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:31,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:33,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:35,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:38,828 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:27:40,537 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:27:40,538 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:27:47,575 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:27:47,575 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:27:47,575 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:27:48,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:27:50,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:52,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:54,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:27:55,669 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:27:55,669 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:27:56,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:07,611 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:28:07,611 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:28:07,612 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:28:08,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:28:08,905 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:28:10,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:10,802 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:28:10,803 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:28:12,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:14,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:16,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:25,935 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:28:25,936 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:28:27,648 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:28:27,648 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:28:27,648 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:28:28,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:28:29,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:30,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:32,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:34,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:36,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:38,981 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:28:41,067 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:28:41,068 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:28:47,688 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:28:47,688 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:28:47,689 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:28:48,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:28:49,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:50,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:52,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:54,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:56,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:28:56,198 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:28:56,198 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:29:07,726 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:29:07,727 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:29:07,727 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:29:08,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:29:09,060 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:29:09,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:10,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:11,330 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:29:11,330 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:29:12,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:14,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:16,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:26,461 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:29:26,461 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:29:27,768 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:29:27,768 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:29:27,768 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:29:28,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:29:29,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:30,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:32,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:34,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:36,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:39,136 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:29:41,593 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:29:41,593 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:29:47,816 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:29:47,817 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:29:47,817 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:29:48,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:29:49,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:50,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:52,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:54,176 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:56,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:29:56,724 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:29:56,724 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:30:07,854 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:30:07,854 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:30:07,854 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:30:08,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:30:09,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:09,211 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:30:10,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:11,853 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:30:11,853 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:30:12,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:14,185 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:16,186 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:26,985 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:30:26,986 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:30:27,892 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:30:27,892 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:30:27,892 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:30:28,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:30:29,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:30,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:32,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:34,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:36,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:39,286 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:30:42,123 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:30:42,123 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:30:47,930 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:30:47,931 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:30:47,931 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:30:48,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:30:49,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:50,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:52,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:54,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:56,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:30:57,256 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:30:57,256 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:31:07,975 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:31:07,976 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:31:07,976 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:31:08,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:31:09,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:09,359 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:31:10,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:12,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:12,396 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:31:12,396 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:31:14,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:16,212 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:27,527 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:31:27,528 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:31:28,016 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:31:28,016 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:31:28,017 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:31:28,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:31:29,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:30,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:32,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:34,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:36,220 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:39,432 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:31:42,660 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:31:42,660 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:31:48,057 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:31:48,057 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:31:48,057 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:31:48,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:31:49,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:50,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:52,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:54,228 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:56,229 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:31:57,793 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:31:57,793 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:32:08,097 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:32:08,097 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:32:08,097 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:32:08,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:32:09,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:09,507 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:32:10,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:12,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:12,935 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:32:12,935 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:32:14,237 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:16,238 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:28,066 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:32:28,067 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:32:28,194 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:32:28,194 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:32:28,195 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:32:28,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:32:29,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:30,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:32,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:34,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:36,246 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:39,581 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:32:43,196 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:32:43,196 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:32:48,180 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:32:48,180 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:32:48,181 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:32:48,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:32:49,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:50,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:52,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:54,254 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:56,255 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:32:58,329 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:32:58,329 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:33:08,217 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:33:08,217 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:33:08,218 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:33:08,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:33:09,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:09,658 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:33:10,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:12,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:13,464 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:33:13,464 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:33:14,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:16,263 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:28,257 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:33:28,257 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:33:28,258 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:33:28,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:33:28,603 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:33:28,604 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:33:29,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:30,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:32,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:34,272 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:36,273 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:39,736 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:33:43,737 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:33:43,737 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:33:48,300 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:33:48,300 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:33:48,300 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:33:49,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:49,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:33:50,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:52,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:54,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:56,281 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:33:58,868 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:33:58,868 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:34:08,353 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:34:08,353 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:34:08,354 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:34:09,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:09,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:34:09,812 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:34:10,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:12,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:14,045 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:34:14,046 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:34:14,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:16,290 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:28,392 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:34:28,392 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:34:28,392 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:34:29,196 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:34:29,196 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:34:29,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:29,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:34:30,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:32,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:34,298 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:36,299 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:39,885 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:34:44,337 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:34:44,337 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:34:48,440 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:34:48,441 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:34:48,441 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:34:49,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:49,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:34:50,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:52,307 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:54,307 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:56,308 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:34:59,474 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:34:59,474 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:35:08,480 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:35:08,480 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:35:08,480 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:35:09,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:35:09,959 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:35:10,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:11,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:12,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:14,316 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:14,604 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:35:14,604 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:35:16,317 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:28,518 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:35:28,518 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:35:28,519 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:35:29,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:29,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:35:29,763 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:35:29,764 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:35:30,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:32,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:34,325 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:36,326 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:40,032 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:35:44,941 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:35:44,942 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:35:48,554 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:35:48,555 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:35:48,555 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:35:49,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:49,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:35:50,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:52,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:54,334 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:35:56,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:00,074 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:36:00,075 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:36:08,593 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:36:08,594 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:36:08,594 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:36:09,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:09,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:36:10,107 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:36:10,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:12,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:14,343 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:15,205 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:36:15,206 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:36:16,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:28,636 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:36:28,636 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:36:28,636 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:36:29,349 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:29,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:36:30,341 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:36:30,342 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:36:30,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:32,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:34,351 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:36,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:40,171 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:36:45,484 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:36:45,484 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:36:48,672 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:36:48,673 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:36:48,673 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:36:49,357 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:49,357 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:36:50,358 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:52,359 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:54,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:36:56,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:00,616 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:37:00,617 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:37:08,958 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:37:08,958 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:37:08,959 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:37:09,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:37:10,243 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:37:10,367 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:12,367 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:14,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:15,745 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:37:15,746 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:37:16,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:28,997 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:37:28,998 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:37:28,998 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:37:29,374 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:37:30,375 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:30,913 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:37:30,913 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:37:32,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:34,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:36,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:40,309 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:37:46,044 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:37:46,045 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:37:49,035 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:37:49,035 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:37:49,035 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:37:49,382 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:37:50,383 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:52,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:54,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:37:56,385 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:01,178 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:38:01,178 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:38:09,073 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:38:09,074 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:38:09,074 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:38:09,390 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:38:10,376 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:38:10,391 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:12,392 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:15,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:16,308 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:38:16,308 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:38:17,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:29,116 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:38:29,117 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:38:29,117 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:38:29,400 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:38:30,401 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:31,401 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:31,441 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:38:31,442 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:38:33,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:35,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:37,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:40,444 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:38:46,574 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:38:46,575 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:38:49,153 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:38:49,154 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:38:49,154 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:38:49,409 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:38:50,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:51,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:53,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:55,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:38:57,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:01,705 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:39:01,705 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:39:09,190 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:39:09,191 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:39:09,191 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:39:09,418 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:39:10,418 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:10,512 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:39:11,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:13,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:15,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:16,838 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:39:16,838 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:39:17,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:29,231 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:39:29,231 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:39:29,232 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:39:29,426 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:39:30,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:31,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:31,971 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:39:31,971 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:39:33,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:35,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:37,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:40,578 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:39:47,138 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:39:47,138 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:39:49,271 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:39:49,272 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:39:49,272 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:39:49,434 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:39:50,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:51,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:53,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:55,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:39:57,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:02,268 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:40:02,268 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:40:09,312 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:40:09,312 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:40:09,313 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:40:09,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:40:10,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:10,649 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:40:11,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:13,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:15,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:17,403 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:40:17,404 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:40:17,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:29,352 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:40:29,352 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:40:29,353 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:40:29,452 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:40:30,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:31,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:32,535 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:40:32,535 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:40:33,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:35,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:37,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:40,721 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:40:47,672 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:40:47,672 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:40:49,397 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:40:49,397 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:40:49,397 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:40:49,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:40:50,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:51,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:53,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:55,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:40:57,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:02,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:41:02,802 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:41:09,441 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:41:09,442 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:41:09,442 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:41:09,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:41:10,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:10,796 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:41:11,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:13,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:15,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:17,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:17,932 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:41:17,932 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:41:29,479 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:41:29,479 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:41:29,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:41:29,481 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:41:30,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:31,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:33,064 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:41:33,064 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:41:33,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:35,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:37,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:40,871 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:41:48,194 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:41:48,195 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:41:49,517 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:41:49,518 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:41:49,518 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:41:50,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:50,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:41:51,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:53,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:55,492 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:41:57,493 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:03,325 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:42:03,326 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:42:09,560 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:42:09,560 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:42:09,561 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:42:10,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:10,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:42:10,945 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:42:11,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:13,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:15,501 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:17,502 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:18,459 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:42:18,459 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:42:29,600 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:42:29,600 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:42:29,600 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:42:30,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:30,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:42:31,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:33,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:33,595 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:42:33,595 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:42:35,510 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:37,511 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:41,018 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:42:48,725 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:42:48,725 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:42:49,644 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:42:49,645 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:42:49,645 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:42:50,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:50,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:42:51,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:53,518 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:55,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:42:57,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:03,864 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:43:03,864 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:43:09,700 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:43:09,701 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:43:09,701 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:43:10,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:10,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:43:11,091 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:43:11,526 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:13,526 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:15,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:17,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:18,995 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:43:18,996 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:43:29,740 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:43:29,740 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:43:29,741 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:43:30,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:30,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:43:31,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:33,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:34,135 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:43:34,135 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:43:35,536 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:37,537 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:41,164 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:43:49,267 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:43:49,267 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:43:49,776 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:43:49,777 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:43:49,777 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:43:50,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:50,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:43:51,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:53,544 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:55,545 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:43:57,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:04,398 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:44:04,398 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:44:09,814 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:44:09,814 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:44:09,815 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:44:10,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:10,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:44:11,237 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:44:11,552 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:13,553 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:15,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:17,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:19,530 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:44:19,531 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:44:29,854 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:44:29,855 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:44:29,855 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:44:30,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:30,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:44:31,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:33,561 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:34,660 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:44:34,660 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:44:35,562 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:37,563 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:41,311 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:44:49,792 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:44:49,792 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:44:49,923 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:44:49,924 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:44:49,924 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:44:50,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:50,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:44:51,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:53,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:55,570 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:44:57,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:04,933 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:45:04,933 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:45:09,929 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:45:09,929 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:45:09,930 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:45:10,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:10,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:45:11,385 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:45:11,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:13,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:15,578 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:17,579 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:20,064 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:45:20,064 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:45:29,969 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:45:29,970 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:45:29,970 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:45:30,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:30,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:45:31,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:33,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:35,203 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:45:35,203 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:45:35,586 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:37,587 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:41,456 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:45:50,043 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:45:50,044 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:45:50,044 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:45:50,344 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:45:50,344 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:45:50,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:45:51,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:53,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:55,595 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:45:57,595 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:05,496 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:46:05,496 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:46:10,082 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:46:10,082 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:46:10,083 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:46:10,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:46:11,531 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:46:11,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:13,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:15,603 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:17,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:20,626 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:46:20,626 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:46:30,122 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:46:30,122 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:46:30,123 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:46:30,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:46:31,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:32,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:34,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:35,757 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:46:35,757 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:46:36,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:38,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:41,604 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:46:50,159 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:46:50,159 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:46:50,160 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:46:50,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:46:50,913 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:46:50,914 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:46:51,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:52,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:54,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:56,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:46:58,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:06,056 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:47:06,056 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:47:10,199 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:47:10,199 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:47:10,200 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:47:10,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:47:11,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:11,679 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:47:12,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:14,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:16,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:18,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:21,187 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:47:21,187 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:47:30,235 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:47:30,235 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:47:30,236 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:47:30,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:47:31,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:32,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:34,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:36,322 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:47:36,323 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:47:36,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:38,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:41,752 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:47:50,276 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:47:50,276 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:47:50,278 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:47:50,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:47:51,469 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:47:51,469 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:47:51,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:52,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:54,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:56,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:47:58,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:06,608 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:48:06,609 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:48:10,315 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:48:10,316 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:48:10,316 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:48:10,653 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:48:11,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:11,828 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:48:12,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:14,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:16,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:18,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:21,740 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:48:21,741 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:48:30,351 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:48:30,352 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:48:30,352 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:48:30,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:48:31,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:32,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:34,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:36,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:36,870 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:48:36,871 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:48:38,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:41,902 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:48:50,386 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:48:50,387 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:48:50,387 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:48:50,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:48:51,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:52,001 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:48:52,002 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:48:52,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:54,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:56,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:48:58,676 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:07,134 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:49:07,134 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:49:10,422 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:49:10,422 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:49:10,423 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:49:10,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:49:11,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:11,976 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:49:12,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:14,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:16,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:18,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:22,269 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:49:22,269 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:49:30,460 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:49:30,460 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:49:30,461 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:49:30,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:49:31,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:32,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:34,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:36,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:37,400 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:49:37,400 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:49:38,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:42,052 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:49:50,499 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:49:50,499 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:49:50,500 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:49:50,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:49:51,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:52,532 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:49:52,533 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:49:52,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:54,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:56,704 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:49:58,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:07,699 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:50:07,699 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:50:10,536 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:50:10,537 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:50:10,537 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:50:10,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:50:11,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:12,125 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:50:12,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:14,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:16,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:18,714 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:22,838 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:50:22,839 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:50:30,573 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:50:30,573 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:50:30,574 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:50:30,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:50:31,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:32,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:34,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:36,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:37,971 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:50:37,971 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:50:38,724 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:42,198 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:50:50,619 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:50:50,619 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:50:50,620 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:50:50,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:50:51,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:52,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:53,121 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:50:53,121 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:50:54,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:56,732 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:50:58,733 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:08,253 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:51:08,253 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:51:10,656 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:51:10,656 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:51:10,657 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:51:10,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:51:11,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:12,272 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:51:12,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:14,741 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:16,742 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:18,743 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:23,385 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:51:23,385 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:51:30,692 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:51:30,692 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:51:30,693 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:51:30,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:51:31,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:32,749 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:34,750 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:36,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:38,518 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:51:38,518 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:51:38,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:42,344 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:51:50,734 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:51:50,734 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:51:50,735 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:51:50,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:51:51,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:52,758 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:53,653 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:51:53,653 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:51:54,759 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:56,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:51:58,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:08,784 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:52:08,785 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:52:10,777 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:52:10,777 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:52:10,778 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:52:11,767 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:11,767 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:52:12,419 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:52:12,768 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:14,768 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:16,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:18,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:23,915 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:52:23,915 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:52:30,831 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:52:30,831 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:52:30,831 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:52:31,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:31,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:52:32,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:34,777 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:36,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:38,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:39,045 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:52:39,045 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:52:42,487 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:52:50,865 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:52:50,865 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:52:50,866 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:52:51,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:51,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:52:52,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:54,179 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:52:54,180 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:52:54,786 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:56,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:52:58,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:09,310 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:53:09,310 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:53:10,902 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:53:10,902 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:53:10,903 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:53:11,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:53:12,554 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:53:12,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:14,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:16,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:18,796 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:24,441 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:53:24,441 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:53:31,235 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:53:31,235 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:53:31,236 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:53:31,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:53:32,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:34,803 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:36,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:38,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:39,580 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:53:39,581 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:53:42,626 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:53:51,275 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:53:51,275 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:53:51,276 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:53:51,811 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:53:52,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:54,715 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:53:54,715 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:53:54,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:56,813 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:53:58,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:09,846 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:54:09,847 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:54:11,325 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:54:11,325 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:54:11,326 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:54:11,820 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:54:12,696 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:54:12,820 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:14,821 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:16,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:18,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:24,977 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:54:24,977 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:54:31,369 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:54:31,370 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:54:31,370 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:54:31,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:54:32,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:34,829 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:37,830 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:39,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:40,113 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:54:40,113 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:54:42,765 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:54:51,408 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:54:51,408 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:54:51,408 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:54:51,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:54:52,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:53,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:55,247 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:54:55,248 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:54:55,838 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:57,839 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:54:59,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:10,383 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:55:10,384 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:55:11,451 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:55:11,452 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:55:11,452 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:55:11,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:55:12,834 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:55:12,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:13,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:15,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:17,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:19,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:25,516 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:55:25,516 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:55:31,491 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:55:31,491 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:55:31,491 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:55:31,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:55:32,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:33,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:35,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:37,857 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:39,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:40,647 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:55:40,648 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:55:42,901 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:55:51,529 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:55:51,530 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:55:51,530 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:55:51,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:55:52,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:53,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:55,787 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:55:55,788 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:55:55,865 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:57,866 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:55:59,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:01,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:03,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:05,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:07,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:09,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:10,918 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:56:10,918 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:56:11,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:12,969 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:56:13,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:15,873 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:17,874 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:19,875 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:21,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:23,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:25,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:26,049 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:56:26,050 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:56:27,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:29,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:31,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:33,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:35,882 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:37,883 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:39,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:41,199 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:56:41,199 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:56:41,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:43,037 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:56:43,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:45,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:47,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:49,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:51,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:55,389 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:56:55,390 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:56:55,390 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:56:55,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:55,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:56:56,350 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:56:56,350 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:56:57,891 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:56:59,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:01,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:03,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:11,482 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:57:11,482 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:57:13,108 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:57:15,457 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:57:15,458 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:57:15,458 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:57:15,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:15,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:57:16,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:18,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:20,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:22,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:24,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:26,612 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:57:26,612 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:57:35,498 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:57:35,498 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:57:35,499 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:57:35,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:57:36,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:38,907 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:40,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:41,746 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:57:41,747 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:57:42,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:43,176 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:57:44,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:55,546 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:57:55,547 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:57:55,547 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:57:55,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:57:56,902 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:57:56,902 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:57:56,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:57:58,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:00,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:02,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:04,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:12,034 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:58:12,034 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:58:13,244 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:58:15,582 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:58:15,583 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:58:15,583 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:58:15,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:58:16,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:18,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:20,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:22,926 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:24,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:27,174 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:58:27,174 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:58:35,621 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:58:35,622 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:58:35,622 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:58:35,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:58:36,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:38,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:40,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:42,307 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:58:42,307 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:58:42,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:43,315 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:58:44,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:55,666 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:58:55,666 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:58:55,666 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:58:55,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:58:56,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:58:57,466 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:58:57,466 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:58:58,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:00,943 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:02,944 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:04,945 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:12,608 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:59:12,608 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:59:13,387 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:59:15,705 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:59:15,705 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:59:15,705 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:59:15,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:59:16,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:18,951 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:20,951 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:22,952 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:24,953 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:27,739 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:59:27,740 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:59:35,744 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:59:35,744 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:59:35,744 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:59:35,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:59:36,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:38,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:40,960 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:42,871 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:59:42,871 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:59:42,961 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:43,458 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 03:59:44,962 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:55,785 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 03:59:55,785 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 03:59:55,786 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 03:59:55,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 03:59:56,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 03:59:58,015 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 03:59:58,015 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 03:59:58,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:00,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:02,969 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:04,970 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:13,156 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:00:13,157 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:00:13,529 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:00:15,832 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:00:15,832 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:00:15,833 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:00:15,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:00:16,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:18,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:20,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:22,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:24,979 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:28,287 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:00:28,288 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:00:35,870 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:00:35,870 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:00:35,871 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:00:35,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:00:36,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:38,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:40,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:42,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:43,426 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:00:43,427 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:00:43,598 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:00:44,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:55,914 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:00:55,914 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:00:55,915 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:00:55,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:00:56,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:00:58,557 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:00:58,557 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:00:58,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:00,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:02,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:04,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:13,669 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:01:13,687 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:01:13,688 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:01:15,952 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:01:15,953 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:01:15,953 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:01:15,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:01:16,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:17,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:19,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:21,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:23,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:25,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:28,818 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:01:28,819 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:01:35,994 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:01:35,994 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:01:35,995 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:01:36,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:01:37,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:38,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:39,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:41,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:43,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:43,742 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:01:43,948 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:01:43,948 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:01:45,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:56,033 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:01:56,034 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:01:56,034 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:01:57,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:57,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:01:58,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:59,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:01:59,083 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:01:59,083 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:02:01,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:03,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:13,816 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:02:14,216 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:02:14,216 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:02:16,074 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:02:16,074 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:02:16,075 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:02:17,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:17,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:02:18,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:19,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:21,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:23,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:26,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:29,347 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:02:29,347 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:02:36,127 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:02:36,128 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:02:36,128 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:02:37,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:02:38,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:40,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:42,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:43,886 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:02:44,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:44,479 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:02:44,479 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:02:46,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:56,166 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:02:56,167 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:02:56,167 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:02:57,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:02:58,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:02:59,608 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:02:59,609 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:03:00,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:02,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:04,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:13,957 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:03:14,742 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:03:14,742 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:03:16,208 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:03:16,208 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:03:16,209 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:03:17,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:03:18,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:20,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:22,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:24,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:29,871 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:03:29,871 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:03:36,247 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:03:36,248 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:03:36,248 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:03:37,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:03:38,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:40,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:42,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:44,030 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:03:44,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:03:45,080 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:03:45,080 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:03:56,288 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:03:56,288 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:03:56,289 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:03:57,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:03:58,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:00,069 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:00,211 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:04:00,211 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:04:02,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:04,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:14,104 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:04:15,349 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:04:15,349 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:04:16,327 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:04:16,327 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:04:16,327 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:04:17,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:04:18,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:20,078 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:22,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:24,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:26,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:30,484 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:04:30,484 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:04:36,366 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:04:36,367 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:04:36,367 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:04:37,085 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:04:38,086 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:40,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:42,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:44,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:44,176 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:04:45,615 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:04:45,615 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:04:46,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:04:56,404 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:04:56,404 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:04:56,405 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:04:57,094 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:04:58,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:00,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:00,745 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:05:00,746 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:05:02,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:04,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:06,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:14,252 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:05:15,899 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:05:15,900 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:05:16,442 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:05:16,442 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:05:16,443 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:05:17,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:05:18,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:20,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:22,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:24,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:26,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:31,031 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:05:31,031 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:05:36,482 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:05:36,483 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:05:36,483 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:05:37,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:05:38,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:40,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:42,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:44,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:44,326 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:05:46,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:05:46,166 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:05:46,166 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:05:56,521 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:05:56,521 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:05:56,521 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:05:57,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:05:58,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:00,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:01,297 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:06:01,297 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:06:02,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:04,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:06,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:14,399 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:06:16,428 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:06:16,428 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:06:16,561 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:06:16,561 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:06:16,561 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:06:17,128 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:06:18,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:20,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:22,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:24,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:26,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:31,599 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:06:31,599 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:06:36,595 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:06:36,595 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:06:36,596 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:06:37,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:06:38,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:40,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:42,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:44,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:44,476 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:06:46,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:06:46,729 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:06:46,729 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:06:56,631 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:06:56,632 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:06:56,632 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:06:57,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:06:58,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:00,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:01,858 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:07:01,859 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:07:02,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:04,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:06,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:14,552 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:07:16,670 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:07:16,671 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:07:16,671 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:07:17,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:07:17,249 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:07:17,250 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:07:18,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:20,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:22,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:24,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:26,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:32,417 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:07:32,418 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:07:36,708 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:07:36,708 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:07:36,709 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:07:37,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:07:38,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:40,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:42,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:44,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:44,626 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:07:46,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:07:47,551 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:07:47,552 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:07:56,746 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:07:56,747 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:07:56,747 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:07:57,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:07:58,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:00,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:02,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:02,798 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:08:02,798 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:08:04,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:06,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:14,702 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:08:16,784 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:08:16,784 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:08:16,785 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:08:17,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:08:17,929 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:08:17,930 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:08:18,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:20,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:22,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:24,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:26,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:33,059 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:08:33,060 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:08:36,822 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:08:36,822 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:08:36,822 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:08:37,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:08:38,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:40,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:42,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:44,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:44,776 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:08:46,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:48,191 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:08:48,192 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:08:56,861 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:08:56,862 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:08:56,862 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:08:57,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:08:58,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:08:59,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:00,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:02,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:03,992 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:09:03,993 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:09:04,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:06,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:14,847 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:09:16,900 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:09:16,901 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:09:16,901 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:09:17,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:09:18,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:19,132 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:09:19,132 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:09:19,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:20,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:22,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:24,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:26,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:34,265 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:09:34,266 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:09:36,944 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:09:36,945 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:09:36,945 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:09:37,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:09:38,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:39,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:40,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:42,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:44,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:44,920 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:09:46,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:49,399 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:09:49,399 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:09:57,027 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:09:57,028 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:09:57,028 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:09:57,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:09:58,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:09:59,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:00,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:02,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:04,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:04,531 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:10:04,532 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:10:06,228 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:14,992 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:10:17,067 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:10:17,068 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:10:17,068 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:10:17,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:10:18,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:19,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:19,665 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:10:19,665 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:10:20,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:22,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:24,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:26,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:34,793 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:10:34,793 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:10:37,107 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:10:37,108 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:10:37,108 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:10:37,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:10:38,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:39,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:40,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:42,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:44,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:45,065 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:10:46,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:49,923 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:10:49,924 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:10:57,143 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:10:57,143 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:10:57,144 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:10:57,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:10:58,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:10:59,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:00,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:02,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:04,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:05,053 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:11:05,054 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:11:06,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:15,138 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:11:17,183 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:11:17,183 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:11:17,183 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:11:17,257 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:11:18,257 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:19,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:20,184 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:11:20,185 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:11:20,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:22,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:24,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:26,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:35,315 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:11:35,316 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:11:37,229 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:11:37,230 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:11:37,230 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:11:37,265 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:11:38,266 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:39,266 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:40,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:42,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:44,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:45,210 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:11:46,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:50,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:11:50,449 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:11:57,268 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:11:57,269 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:11:57,269 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:11:57,274 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:11:58,275 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:11:59,275 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:00,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:02,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:04,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:05,588 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:12:05,589 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:12:06,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:15,285 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:12:17,314 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:12:17,314 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:12:17,314 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:12:18,283 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:18,283 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:12:19,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:20,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:20,722 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:12:20,723 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:12:22,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:24,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:27,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:35,854 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:12:35,854 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:12:37,360 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:12:37,360 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:12:37,361 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:12:38,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:12:39,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:41,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:43,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:45,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:45,359 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:12:47,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:12:50,983 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:12:50,984 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:12:57,413 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:12:57,414 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:12:57,414 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:12:58,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:12:59,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:01,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:03,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:05,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:06,116 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:13:06,117 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:13:07,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:15,434 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:13:17,456 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:13:17,456 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:13:17,456 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:13:18,310 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:13:19,310 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:21,249 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:13:21,249 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:13:21,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:23,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:25,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:27,318 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:36,379 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:13:36,380 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:13:37,500 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:13:37,501 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:13:37,501 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:13:38,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:13:39,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:41,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:43,325 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:45,326 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:45,508 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:13:47,327 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:13:51,514 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:13:51,514 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:13:57,539 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:13:57,540 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:13:57,540 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:13:58,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:13:59,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:01,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:03,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:05,334 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:06,652 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:14:06,652 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:14:07,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:15,583 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:14:17,579 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:14:17,579 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:14:17,580 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:14:18,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:14:19,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:21,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:21,785 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:14:21,785 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:14:23,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:25,343 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:27,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:36,917 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:14:36,918 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:14:37,619 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:14:37,619 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:14:37,619 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:14:38,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:14:39,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:41,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:43,355 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:45,356 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:45,659 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:14:47,357 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:14:52,052 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:14:52,052 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:14:57,657 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:14:57,658 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:14:57,658 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:14:58,362 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:14:59,362 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:01,363 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:03,364 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:05,365 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:07,185 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:15:07,186 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:15:07,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:15,735 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:15:17,696 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:15:17,697 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:15:17,697 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:15:18,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:15:19,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:21,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:22,318 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:15:22,318 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:15:23,372 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:25,373 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:37,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:15:37,448 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:15:37,736 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:15:37,736 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:15:37,736 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:15:38,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:15:39,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:41,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:43,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:45,381 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:15:45,812 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:15:52,579 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:15:52,580 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:15:57,774 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:15:57,775 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:15:57,775 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:15:58,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:15:59,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:01,388 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:03,389 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:05,390 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:07,719 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:16:07,719 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:16:15,887 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:16:17,821 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:16:17,822 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:16:17,822 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:16:18,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:16:19,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:21,396 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:22,850 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:16:22,850 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:16:23,397 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:25,398 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:37,862 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:16:37,862 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:16:37,863 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:16:37,987 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:16:37,988 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:16:38,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:16:39,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:41,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:43,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:45,406 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:16:45,963 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:16:53,120 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:16:53,120 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:16:57,902 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:16:57,902 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:16:57,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:16:58,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:16:59,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:01,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:03,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:05,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:08,250 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:17:08,251 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:17:16,037 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:17:17,937 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:17:17,938 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:17:17,939 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:17:18,418 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:17:19,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:21,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:23,380 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:17:23,381 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:17:23,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:25,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:37,976 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:17:37,976 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:17:37,977 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:17:38,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:17:38,543 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:17:38,543 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:17:39,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:41,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:43,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:45,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:17:46,114 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:17:53,683 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:17:53,684 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:17:58,020 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:17:58,021 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:17:58,021 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:17:58,434 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:17:59,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:00,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:01,435 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:03,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:05,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:08,827 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:18:08,828 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:18:16,188 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:18:18,059 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:18:18,060 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:18:18,060 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:18:18,442 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:18:19,443 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:20,443 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:21,443 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:23,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:23,957 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:18:23,957 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:18:25,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:38,109 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:18:38,110 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:18:38,110 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:18:38,450 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:18:39,111 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:18:39,111 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:18:39,451 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:40,451 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:41,452 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:43,452 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:45,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:18:46,265 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:18:54,253 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:18:54,253 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:18:58,147 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:18:58,147 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:18:58,148 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:18:58,459 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:18:59,459 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:00,460 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:01,460 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:03,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:05,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:07,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:09,388 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:19:09,388 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:19:16,339 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:19:18,189 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:19:18,189 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:19:18,189 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:19:18,467 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:19:19,467 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:20,468 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:21,468 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:23,469 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:24,518 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:19:24,518 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:19:25,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:27,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:38,231 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:19:38,231 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:19:38,231 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:19:38,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:19:39,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:39,686 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:19:39,686 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:19:40,476 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:41,476 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:43,477 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:45,478 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:46,413 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:19:47,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:19:54,816 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:19:54,817 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:19:58,276 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:19:58,276 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:19:58,276 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:19:58,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:19:59,484 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:00,487 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:01,484 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:03,485 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:05,486 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:09,946 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:20:09,947 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:20:16,487 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:20:18,314 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:20:18,315 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:20:18,315 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:20:18,492 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:20:20,493 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:22,493 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:24,494 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:25,077 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:20:25,077 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:20:26,495 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:38,359 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:20:38,360 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:20:38,360 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:20:38,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:20:40,230 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:20:40,230 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:20:40,501 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:42,502 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:44,503 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:46,504 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:20:46,566 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:20:55,371 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:20:55,371 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:20:58,399 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:20:58,400 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:20:58,401 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:20:58,511 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:21:00,511 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:02,512 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:04,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:06,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:10,511 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:21:10,512 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:21:16,643 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:21:18,458 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:21:18,458 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:21:18,459 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:21:18,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:21:20,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:22,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:24,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:25,645 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:21:25,646 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:21:26,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:38,489 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:21:38,489 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:21:38,490 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:21:38,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:21:40,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:40,778 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:21:40,778 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:21:42,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:44,529 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:46,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:21:46,717 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:21:55,909 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:21:55,910 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:21:58,528 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:21:58,528 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:21:58,528 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:21:58,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:22:00,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:02,536 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:04,537 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:06,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:11,046 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:22:11,047 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:22:16,784 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:22:18,565 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:22:18,565 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:22:18,566 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:22:19,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:22:20,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:22,544 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:24,545 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:26,184 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:22:26,184 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:22:26,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:38,604 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:22:38,604 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:22:38,605 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:22:39,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:22:40,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:41,313 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:22:41,314 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:22:42,552 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:44,553 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:46,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:22:46,848 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:22:56,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:22:56,448 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:22:58,644 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:22:58,644 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:22:58,645 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:22:59,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:23:00,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:02,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:04,561 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:06,562 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:11,583 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:23:11,584 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:23:16,912 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:23:18,683 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:23:18,684 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:23:18,684 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:23:19,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:23:20,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:22,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:24,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:26,570 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:26,715 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:23:26,716 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:23:38,723 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:23:38,723 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:23:38,724 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:23:39,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:23:40,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:41,847 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:23:41,847 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:23:42,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:44,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:46,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:23:46,979 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:23:56,978 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:23:56,978 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:23:58,764 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:23:58,764 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:23:58,765 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:23:59,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:24:00,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:02,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:04,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:06,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:12,113 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:24:12,114 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:24:17,046 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:24:18,804 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:24:18,805 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:24:18,805 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:24:19,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:24:20,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:22,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:24,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:26,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:27,245 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:24:27,245 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:24:38,843 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:24:38,843 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:24:38,844 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:24:39,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:24:40,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:42,377 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:24:42,378 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:24:42,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:44,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:46,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:24:47,113 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:24:57,510 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:24:57,511 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:24:58,886 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:24:58,886 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:24:58,887 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:24:59,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:25:00,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:02,608 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:04,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:06,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:12,642 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:25:12,643 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:25:17,180 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:25:18,925 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:25:18,925 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:25:18,926 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:25:19,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:25:20,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:22,616 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:24,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:26,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:27,773 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:25:27,773 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:25:38,965 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:25:38,966 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:25:38,966 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:25:39,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:25:40,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:42,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:42,911 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:25:42,911 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:25:44,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:46,626 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:25:47,253 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:25:58,064 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:25:58,065 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:25:59,003 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:25:59,004 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:25:59,004 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:25:59,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:26:00,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:02,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:04,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:06,634 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:13,195 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:26:13,196 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:26:17,320 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:26:19,045 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:26:19,045 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:26:19,046 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:26:19,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:26:20,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:22,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:24,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:26,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:28,326 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:26:28,326 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:26:39,085 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:26:39,086 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:26:39,086 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:26:39,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:26:40,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:42,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:43,465 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:26:43,465 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:26:44,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:46,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:26:47,389 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:26:58,599 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:26:58,600 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:26:59,126 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:26:59,127 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:26:59,127 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:26:59,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:27:00,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:02,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:04,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:06,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:13,730 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:27:13,730 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:27:17,459 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:27:19,172 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:27:19,172 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:27:19,173 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:27:19,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:27:20,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:22,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:24,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:26,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:28,860 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:27:28,860 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:27:39,212 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:27:39,212 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:27:39,213 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:27:39,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:27:41,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:43,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:44,068 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:27:44,069 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:27:45,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:47,527 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:27:47,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:27:59,200 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:27:59,201 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:27:59,332 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:27:59,333 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:27:59,333 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:27:59,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:28:01,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:03,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:05,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:07,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:14,362 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:28:14,363 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:28:17,596 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:28:19,291 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:28:19,291 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:28:19,292 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:28:19,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:28:21,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:23,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:25,690 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:27,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:29,493 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:28:29,494 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:28:39,331 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:28:39,331 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:28:39,332 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:28:39,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:28:41,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:43,698 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:44,635 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:28:44,636 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:28:45,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:47,664 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:28:47,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:28:59,373 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:28:59,373 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:28:59,373 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:28:59,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:28:59,769 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:28:59,769 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:29:01,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:03,706 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:05,707 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:07,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:14,933 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:29:14,933 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:29:17,730 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:29:19,412 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:29:19,413 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:29:19,413 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:29:19,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:29:21,714 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:23,715 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:25,716 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:27,716 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:30,066 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:29:30,066 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:29:39,461 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:29:39,461 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:29:39,462 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:29:39,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:29:41,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:43,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:45,198 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:29:45,198 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:29:45,724 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:47,725 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:29:47,798 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:29:59,502 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:29:59,502 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:29:59,503 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:29:59,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:30:00,336 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:30:00,337 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:30:01,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:03,732 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:05,733 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:07,734 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:15,472 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:30:15,473 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:30:17,864 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:30:19,542 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:30:19,542 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:30:19,543 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:30:19,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:30:21,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:23,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:25,741 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:27,742 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:30,604 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:30:30,604 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:30:39,584 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:30:39,584 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:30:39,585 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:30:39,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:30:41,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:43,749 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:45,747 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:30:45,748 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:30:45,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:47,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:30:47,931 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:30:59,623 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:30:59,624 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:30:59,624 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:30:59,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:31:00,879 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:31:00,880 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:31:01,758 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:03,758 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:05,759 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:07,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:16,016 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:31:16,016 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:31:18,001 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:31:19,669 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:31:19,670 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:31:19,670 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:31:19,765 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:31:21,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:23,767 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:25,768 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:27,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:31,149 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:31:31,150 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:31:39,708 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:31:39,709 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:31:39,709 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:31:39,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:31:41,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:43,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:45,777 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:46,278 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:31:46,278 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:31:47,777 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:31:48,068 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:31:59,746 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:31:59,746 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:31:59,747 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:31:59,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:32:01,444 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:32:01,445 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:32:01,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:03,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:05,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:07,786 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:16,586 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:32:16,587 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:32:18,138 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:32:19,786 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:32:19,786 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:32:19,787 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:32:19,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:32:21,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:23,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:25,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:27,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:31,717 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:32:31,717 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:32:39,825 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:32:39,826 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:32:39,826 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:32:40,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:32:41,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:43,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:45,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:46,849 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:32:46,849 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:32:47,803 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:32:48,204 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:32:59,864 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:32:59,865 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:32:59,865 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:33:00,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:33:01,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:01,979 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:33:01,980 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:33:03,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:05,811 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:07,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:17,702 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:33:17,703 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:33:18,279 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:33:19,904 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:33:19,904 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:33:19,905 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:33:20,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:33:21,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:23,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:25,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:27,820 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:32,845 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:33:32,846 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:33:39,944 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:33:39,945 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:33:39,945 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:33:40,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:33:41,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:43,827 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:45,827 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:47,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:33:47,977 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:33:47,977 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:33:48,355 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:33:59,985 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:33:59,985 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:33:59,986 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:34:00,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:34:01,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:03,108 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:34:03,108 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:34:03,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:05,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:07,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:18,239 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:34:18,240 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:34:18,430 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:34:20,026 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:34:20,027 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:34:20,027 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:34:20,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:34:21,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:23,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:25,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:27,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:33,371 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:34:33,371 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:34:40,069 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:34:40,069 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:34:40,069 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:34:40,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:34:41,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:43,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:45,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:47,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:34:48,502 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:34:48,511 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:34:48,512 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:35:00,115 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:35:00,116 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:35:00,116 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:35:00,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:35:01,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:03,642 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:35:03,642 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:35:03,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:05,860 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:07,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:18,576 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:35:18,779 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:35:18,779 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:35:20,162 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:35:20,162 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:35:20,163 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:35:20,866 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:35:21,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:23,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:25,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:27,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:33,918 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:35:33,918 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:35:40,205 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:35:40,206 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:35:40,206 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:35:40,875 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:35:41,875 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:43,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:45,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:47,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:35:48,647 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:35:49,047 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:35:49,048 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:36:00,253 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:36:00,254 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:36:00,254 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:36:00,883 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:36:01,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:03,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:04,178 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:36:04,178 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:36:05,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:07,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:18,722 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:36:19,315 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:36:19,315 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:36:20,291 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:36:20,291 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:36:20,292 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:36:20,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:36:21,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:23,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:25,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:27,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:34,446 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:36:34,446 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:36:40,333 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:36:40,334 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:36:40,334 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:36:40,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:36:41,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:43,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:45,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:47,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:36:48,795 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:36:49,583 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:36:49,584 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:37:00,380 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:37:00,380 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:37:00,381 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:37:00,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:37:01,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:03,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:04,745 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:37:04,745 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:37:05,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:07,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:10,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:12,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:14,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:16,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:18,866 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:37:18,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:19,879 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:37:19,879 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:37:20,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:22,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:24,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:26,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:28,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:30,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:32,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:34,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:35,015 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:37:35,016 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:37:36,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:38,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:40,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:42,926 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:44,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:46,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:48,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:48,939 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:37:50,168 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:37:50,168 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:37:50,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:52,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:54,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:56,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:37:58,934 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:38:00,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:38:04,239 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:38:04,239 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:38:04,240 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:38:04,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:38:05,300 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:38:05,301 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:38:11,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:38:14,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:38:16,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:38:19,012 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:38:20,432 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:38:20,571 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status