diff --git "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" --- "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" +++ "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" @@ -7754,3 +7754,3754 @@ 2021-07-16 04:38:19,012 DEBUG SenderThread:761704 [sender.py:send():179] send: stats 2021-07-16 04:38:20,432 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status 2021-07-16 04:38:20,571 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:38:35,844 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:38:35,845 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:38:49,084 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:38:50,976 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:38:50,977 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:39:06,112 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:39:06,113 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:39:18,969 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:19,158 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:39:20,970 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:21,243 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:39:21,243 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:39:22,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:24,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:26,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:36,377 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:39:36,378 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:39:38,318 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:39:38,319 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:39:38,319 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:39:38,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:38,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:39:40,979 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:42,979 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:44,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:46,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:49,234 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:39:51,510 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:39:51,510 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:39:58,366 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:39:58,367 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:39:58,367 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:39:58,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:39:58,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:40:00,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:02,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:04,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:06,640 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:40:06,640 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:40:06,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:18,405 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:40:18,405 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:40:18,406 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:40:18,996 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:40:19,310 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:40:20,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:21,772 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:40:21,772 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:40:22,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:24,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:27,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:36,904 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:40:36,904 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:40:38,442 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:40:38,442 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:40:38,443 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:40:39,005 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:39,005 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:40:41,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:43,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:45,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:47,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:49,384 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:40:52,034 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:40:52,034 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:40:58,484 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:40:58,485 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:40:58,485 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:40:59,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:40:59,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:41:01,015 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:03,015 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:05,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:07,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:07,166 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:41:07,167 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:41:18,522 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:41:18,522 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:41:18,522 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:41:19,022 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:41:19,458 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:41:20,023 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:22,023 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:22,298 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:41:22,299 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:41:24,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:26,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:28,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:37,428 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:41:37,429 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:41:38,565 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:41:38,565 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:41:38,566 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:41:39,030 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:41:40,031 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:42,032 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:44,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:46,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:48,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:41:49,533 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:41:52,563 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:41:52,564 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:41:58,603 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:41:58,604 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:41:58,604 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:41:59,039 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:42:00,040 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:02,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:04,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:06,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:07,694 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:42:07,694 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:42:08,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:18,645 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:42:18,645 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:42:18,646 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:42:19,048 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:42:19,606 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:42:20,049 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:22,049 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:22,828 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:42:22,829 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:42:24,050 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:26,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:28,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:37,957 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:42:37,958 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:42:38,689 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:42:38,689 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:42:38,689 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:42:39,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:42:40,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:42,058 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:44,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:46,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:48,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:42:49,680 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:42:53,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:42:53,089 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:42:58,729 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:42:58,729 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:42:58,730 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:42:59,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:43:00,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:02,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:04,067 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:06,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:08,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:08,219 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:43:08,219 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:43:18,768 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:43:18,768 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:43:18,768 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:43:19,073 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:43:19,752 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:43:20,074 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:22,074 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:23,351 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:43:23,351 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:43:24,075 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:26,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:28,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:38,481 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:43:38,481 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:43:38,812 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:43:38,812 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:43:38,813 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:43:39,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:43:40,082 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:42,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:44,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:46,084 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:48,085 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:43:49,825 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:43:53,613 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:43:53,613 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:43:58,850 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:43:58,851 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:43:58,851 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:43:59,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:44:00,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:02,091 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:04,092 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:06,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:08,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:08,743 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:44:08,744 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:44:18,889 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:44:18,890 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:44:18,890 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:44:19,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:44:19,891 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:44:20,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:22,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:23,904 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:44:23,905 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:44:24,100 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:26,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:28,102 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:38,927 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:44:38,927 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:44:38,927 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:44:39,051 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:44:39,051 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:44:39,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:44:40,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:42,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:44,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:46,110 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:48,111 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:44:49,966 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:44:54,188 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:44:54,188 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:44:58,964 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:44:58,964 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:44:58,964 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:44:59,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:45:00,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:02,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:04,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:06,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:08,119 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:09,319 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:45:09,319 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:45:19,006 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:45:19,006 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:45:19,007 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:45:19,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:45:20,041 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:45:20,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:22,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:24,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:24,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:45:24,448 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:45:26,126 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:28,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:39,044 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:45:39,045 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:45:39,045 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:45:39,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:45:39,598 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:45:39,598 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:45:40,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:42,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:44,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:46,134 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:48,135 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:45:50,112 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:45:54,739 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:45:54,739 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:45:59,081 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:45:59,081 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:45:59,082 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:45:59,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:46:00,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:02,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:04,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:06,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:08,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:09,870 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:46:09,870 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:46:19,122 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:46:19,123 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:46:19,123 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:46:19,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:46:20,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:20,186 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:46:22,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:24,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:25,001 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:46:25,002 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:46:26,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:28,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:39,160 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:46:39,161 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:46:39,161 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:46:40,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:40,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:46:40,157 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:46:40,157 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:46:42,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:44,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:46,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:48,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:46:50,259 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:46:55,289 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:46:55,290 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:46:59,205 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:46:59,205 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:46:59,206 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:47:00,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:00,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:47:02,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:04,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:06,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:08,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:10,419 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:47:10,420 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:47:19,241 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:47:19,241 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:47:19,242 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:47:20,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:20,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:47:20,334 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:47:22,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:24,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:25,549 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:47:25,549 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:47:26,176 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:28,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:39,281 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:47:39,282 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:47:39,282 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:47:40,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:40,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:47:40,711 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:47:40,712 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:47:42,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:44,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:46,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:48,185 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:47:50,408 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:47:55,850 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:47:55,851 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:47:59,323 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:47:59,324 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:47:59,324 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:48:00,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:00,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:48:02,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:04,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:06,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:08,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:10,980 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:48:10,981 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:48:19,369 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:48:19,369 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:48:19,369 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:48:20,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:20,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:48:20,482 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:48:22,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:24,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:26,114 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:48:26,114 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:48:26,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:28,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:39,409 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:48:39,409 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:48:39,410 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:48:40,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:40,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:48:41,436 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:48:41,436 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:48:42,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:44,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:46,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:48,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:48:50,551 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:48:56,601 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:48:56,601 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:48:59,448 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:48:59,448 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:48:59,449 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:49:00,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:00,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:49:02,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:04,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:06,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:08,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:11,732 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:49:11,732 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:49:19,485 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:49:19,486 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:49:19,486 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:49:20,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:20,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:49:20,623 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:49:22,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:24,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:26,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:26,879 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:49:26,879 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:49:28,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:39,520 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:49:39,521 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:49:39,521 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:49:40,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:40,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:49:42,053 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:49:42,053 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:49:42,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:45,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:47,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:49,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:49:50,698 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:49:57,183 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:49:57,184 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:49:59,793 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:49:59,794 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:49:59,794 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:50:00,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:00,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:50:01,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:03,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:05,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:07,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:09,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:12,316 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:50:12,316 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:50:19,832 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:50:19,832 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:50:19,832 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:50:20,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:50:20,773 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:50:21,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:23,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:25,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:27,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:27,447 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:50:27,447 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:50:29,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:39,871 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:50:39,871 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:50:39,871 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:50:40,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:50:41,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:42,580 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:50:42,580 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:50:43,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:45,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:47,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:49,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:50:50,848 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:50:57,711 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:50:57,711 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:50:59,911 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:50:59,911 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:50:59,912 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:51:00,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:51:01,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:03,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:05,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:07,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:09,271 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:12,843 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:51:12,843 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:51:19,950 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:51:19,950 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:51:19,950 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:51:20,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:51:20,922 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:51:21,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:23,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:25,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:27,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:27,976 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:51:27,976 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:51:29,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:39,989 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:51:39,990 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:51:39,990 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:51:40,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:51:41,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:43,107 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:51:43,107 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:51:43,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:45,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:47,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:49,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:51:50,999 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:51:58,237 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:51:58,237 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:52:00,033 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:52:00,033 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:52:00,034 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:52:00,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:52:01,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:03,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:05,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:07,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:09,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:13,366 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:52:13,367 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:52:20,075 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:52:20,075 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:52:20,075 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:52:20,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:52:21,074 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:52:21,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:23,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:25,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:27,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:28,498 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:52:28,498 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:52:29,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:40,112 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:52:40,112 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:52:40,113 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:52:40,310 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:52:41,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:43,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:43,654 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:52:43,654 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:52:45,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:47,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:49,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:52:51,147 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:52:58,811 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:52:58,811 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:53:00,150 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:53:00,151 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:53:00,151 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:53:00,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:53:01,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:03,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:05,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:07,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:09,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:13,944 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:53:13,944 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:53:20,190 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:53:20,190 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:53:20,191 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:53:20,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:53:21,221 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:53:21,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:23,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:25,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:27,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:29,074 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:53:29,074 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:53:29,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:40,228 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:53:40,228 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:53:40,228 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:53:40,336 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:53:41,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:43,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:44,205 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:53:44,206 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:53:45,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:47,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:49,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:53:51,293 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:53:59,340 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:53:59,340 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:54:00,264 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:54:00,264 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:54:00,265 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:54:00,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:54:01,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:03,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:05,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:07,347 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:09,348 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:14,473 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:54:14,474 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:54:20,304 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:54:20,304 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:54:20,305 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:54:20,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:54:21,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:21,364 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:54:23,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:25,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:27,355 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:29,356 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:29,605 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:54:29,605 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:54:40,343 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:54:40,343 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:54:40,344 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:54:40,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:54:41,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:43,362 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:44,737 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:54:44,737 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:54:45,363 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:47,364 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:49,364 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:54:51,437 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:54:59,869 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:54:59,870 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:55:00,385 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:55:00,386 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:55:00,386 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:55:01,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:01,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:55:03,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:05,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:07,372 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:09,373 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:15,001 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:55:15,002 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:55:20,436 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:55:20,436 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:55:20,437 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:55:21,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:21,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:55:21,511 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:55:23,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:25,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:27,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:29,381 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:30,134 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:55:30,135 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:55:40,480 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:55:40,481 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:55:40,481 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:55:41,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:41,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:55:43,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:45,266 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:55:45,266 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:55:45,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:47,388 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:49,389 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:55:51,587 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:56:00,397 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:56:00,397 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:56:00,526 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:56:00,526 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:56:00,527 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:56:01,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:01,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:56:03,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:05,396 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:07,396 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:09,397 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:15,558 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:56:15,558 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:56:20,562 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:56:20,562 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:56:20,562 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:56:21,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:21,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:56:21,661 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:56:23,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:25,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:27,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:29,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:30,690 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:56:30,690 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:56:40,600 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:56:40,600 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:56:40,601 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:56:41,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:41,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:56:44,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:45,820 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:56:45,820 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:56:46,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:48,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:50,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:56:51,734 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:57:00,638 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:57:00,639 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:57:00,639 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:57:01,072 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:57:01,072 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:57:01,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:57:02,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:04,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:06,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:08,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:10,423 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:16,240 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:57:16,240 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:57:20,674 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:57:20,675 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:57:20,675 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:57:21,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:57:21,806 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:57:22,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:24,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:26,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:28,431 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:30,432 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:31,378 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:57:31,378 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:57:40,712 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:57:40,712 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:57:40,713 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:57:41,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:57:42,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:44,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:46,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:46,521 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:57:46,521 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:57:48,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:50,441 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:57:51,879 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:58:00,749 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:58:00,749 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:58:00,749 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:58:01,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:58:01,682 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:58:01,682 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:58:02,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:04,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:06,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:08,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:10,449 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:16,823 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:58:16,824 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:58:20,785 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:58:20,785 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:58:20,786 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:58:21,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:58:21,947 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:58:22,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:24,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:26,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:28,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:30,457 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:31,954 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:58:31,954 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:58:40,821 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:58:40,821 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:58:40,822 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:58:41,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:58:42,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:44,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:46,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:47,084 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:58:47,085 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:58:48,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:50,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:58:52,021 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:59:00,858 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:59:00,858 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:59:00,859 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:59:01,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:59:02,229 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:59:02,230 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:59:02,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:04,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:06,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:08,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:10,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:17,359 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:59:17,359 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:59:20,899 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:59:20,900 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:59:20,900 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:59:21,478 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:59:22,096 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 04:59:22,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:24,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:26,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:28,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:30,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:32,497 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:59:32,497 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:59:40,942 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 04:59:40,942 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 04:59:40,943 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 04:59:41,487 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 04:59:42,487 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:44,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:46,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:47,638 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 04:59:47,638 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 04:59:48,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:50,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 04:59:52,172 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:00:00,982 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:00:00,983 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:00:00,983 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:00:01,495 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:00:02,496 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:02,781 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:00:02,782 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:00:04,496 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:06,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:08,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:10,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:17,914 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:00:17,914 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:00:21,020 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:00:21,021 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:00:21,021 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:00:21,503 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:00:22,250 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:00:22,504 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:24,504 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:26,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:28,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:30,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:33,045 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:00:33,046 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:00:41,064 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:00:41,065 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:00:41,065 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:00:41,511 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:00:42,512 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:44,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:46,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:48,183 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:00:48,183 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:00:48,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:50,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:00:52,324 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:01:01,106 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:01:01,107 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:01:01,107 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:01:01,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:01:02,520 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:03,346 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:01:03,346 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:01:04,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:06,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:08,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:10,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:18,476 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:01:18,477 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:01:21,150 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:01:21,151 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:01:21,151 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:01:21,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:01:22,399 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:01:22,528 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:24,529 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:26,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:28,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:30,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:33,610 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:01:33,610 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:01:41,188 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:01:41,189 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:01:41,189 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:01:41,536 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:01:42,537 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:44,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:46,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:48,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:48,741 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:01:48,741 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:01:50,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:01:52,474 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:02:01,229 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:02:01,230 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:02:01,230 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:02:01,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:02:02,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:03,872 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:02:03,873 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:02:04,547 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:06,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:08,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:10,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:19,004 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:02:19,004 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:02:21,277 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:02:21,277 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:02:21,277 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:02:21,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:02:22,547 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:02:22,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:24,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:26,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:28,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:30,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:34,133 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:02:34,133 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:02:41,321 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:02:41,321 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:02:41,322 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:02:41,564 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:02:42,564 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:44,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:46,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:48,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:49,263 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:02:49,263 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:02:50,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:02:52,623 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:03:01,363 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:03:01,363 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:03:01,364 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:03:01,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:03:02,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:04,395 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:03:04,395 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:03:04,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:06,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:08,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:10,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:19,526 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:03:19,527 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:03:21,405 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:03:21,405 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:03:21,406 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:03:21,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:03:22,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:22,696 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:03:24,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:26,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:28,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:30,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:34,659 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:03:34,659 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:03:41,444 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:03:41,444 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:03:41,445 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:03:41,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:03:42,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:44,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:46,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:48,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:49,787 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:03:49,788 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:03:50,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:03:52,770 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:04:01,480 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:04:01,481 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:04:01,481 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:04:01,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:04:02,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:04,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:04,923 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:04:04,924 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:04:06,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:08,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:10,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:20,053 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:04:20,053 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:04:21,529 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:04:21,529 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:04:21,530 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:04:21,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:04:22,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:22,843 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:04:24,608 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:26,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:28,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:30,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:35,184 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:04:35,185 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:04:41,571 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:04:41,572 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:04:41,572 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:04:41,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:04:42,616 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:44,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:46,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:48,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:50,318 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:04:50,318 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:04:50,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:04:52,917 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:05:01,615 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:05:01,615 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:05:01,616 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:05:01,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:05:02,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:04,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:05,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:05:05,448 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:05:06,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:08,626 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:10,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:20,582 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:05:20,583 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:05:21,655 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:05:21,655 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:05:21,655 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:05:22,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:22,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:05:22,989 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:05:24,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:26,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:28,634 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:30,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:35,711 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:05:35,711 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:05:41,694 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:05:41,694 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:05:41,695 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:05:42,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:42,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:05:44,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:46,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:48,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:50,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:05:50,843 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:05:50,844 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:05:53,062 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:06:01,731 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:06:01,731 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:06:01,732 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:06:02,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:02,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:06:03,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:05,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:05,975 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:06:05,975 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:06:07,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:09,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:11,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:21,114 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:06:21,115 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:06:21,778 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:06:21,778 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:06:21,778 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:06:22,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:06:23,130 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:06:23,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:25,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:27,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:29,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:31,659 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:36,247 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:06:36,247 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:06:41,832 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:06:41,832 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:06:41,832 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:06:42,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:06:43,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:45,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:47,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:49,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:51,377 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:06:51,377 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:06:51,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:06:53,196 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:07:01,877 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:07:01,877 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:07:01,878 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:07:02,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:07:03,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:05,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:06,507 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:07:06,507 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:07:07,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:09,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:11,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:21,638 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:07:21,638 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:07:21,917 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:07:21,917 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:07:21,918 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:07:22,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:07:23,263 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:07:23,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:25,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:27,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:29,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:31,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:36,774 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:07:36,775 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:07:41,959 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:07:41,960 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:07:41,960 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:07:42,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:07:43,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:45,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:47,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:49,690 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:07:51,940 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:07:51,940 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:07:53,332 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:08:01,999 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:08:01,999 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:08:02,000 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:08:02,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:08:03,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:05,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:07,073 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:08:07,074 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:08:07,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:09,698 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:22,043 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:08:22,044 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:08:22,044 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:08:22,219 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:08:22,219 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:08:22,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:08:23,405 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:08:23,704 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:25,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:27,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:29,706 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:37,360 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:08:37,360 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:08:42,084 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:08:42,085 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:08:42,085 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:08:42,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:08:43,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:45,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:47,714 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:49,715 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:08:52,497 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:08:52,498 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:08:53,473 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:09:02,133 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:09:02,133 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:09:02,134 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:09:02,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:09:03,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:05,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:07,628 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:09:07,628 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:09:07,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:09,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:22,174 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:09:22,175 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:09:22,175 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:09:22,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:09:22,787 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:09:22,788 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:09:23,541 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:09:23,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:25,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:27,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:29,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:37,919 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:09:37,920 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:09:42,213 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:09:42,214 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:09:42,214 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:09:42,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:09:43,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:45,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:47,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:49,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:09:53,061 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:09:53,061 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:09:53,607 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:10:02,255 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:10:02,255 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:10:02,255 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:10:02,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:10:03,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:05,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:07,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:08,191 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:10:08,191 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:10:09,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:22,294 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:10:22,295 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:10:22,295 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:10:22,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:10:23,358 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:10:23,358 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:10:23,673 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:10:23,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:25,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:27,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:29,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:38,499 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:10:38,499 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:10:42,338 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:10:42,338 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:10:42,339 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:10:42,759 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:10:43,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:45,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:47,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:49,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:10:53,628 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:10:53,629 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:10:53,767 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:11:02,384 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:11:02,385 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:11:02,385 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:11:02,767 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:11:03,767 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:05,768 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:07,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:08,770 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:11:08,770 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:11:09,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:22,429 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:11:22,430 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:11:22,430 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:11:22,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:11:23,812 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:11:23,910 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:11:23,911 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:11:24,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:26,777 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:28,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:30,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:39,083 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:11:39,083 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:11:42,468 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:11:42,468 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:11:42,469 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:11:42,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:11:44,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:46,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:48,786 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:50,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:11:53,884 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:11:54,215 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:11:54,215 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:12:02,505 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:12:02,506 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:12:02,506 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:12:02,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:12:04,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:06,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:08,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:09,356 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:12:09,357 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:12:10,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:22,545 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:12:22,546 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:12:22,546 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:12:22,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:12:23,955 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:12:24,664 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:12:24,664 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:12:24,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:26,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:28,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:30,803 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:39,797 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:12:39,798 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:12:42,584 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:12:42,585 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:12:42,585 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:12:42,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:12:44,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:46,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:48,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:50,811 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:12:54,023 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:12:54,930 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:12:54,930 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:13:02,623 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:13:02,623 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:13:02,623 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:13:02,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:13:04,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:06,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:08,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:10,062 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:13:10,062 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:13:10,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:22,660 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:13:22,660 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:13:22,661 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:13:22,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:13:24,091 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:13:24,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:25,213 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:13:25,214 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:13:26,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:28,827 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:30,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:40,352 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:13:40,352 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:13:42,697 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:13:42,698 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:13:42,698 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:13:42,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:13:44,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:46,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:48,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:50,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:13:54,163 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:13:55,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:13:55,488 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:14:02,734 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:14:02,735 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:14:02,735 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:14:02,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:14:04,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:06,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:08,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:10,620 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:14:10,620 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:14:10,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:22,773 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:14:22,773 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:14:22,774 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:14:22,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:14:24,232 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:14:24,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:25,753 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:14:25,754 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:14:26,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:28,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:30,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:40,884 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:14:40,884 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:14:42,809 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:14:42,809 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:14:42,809 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:14:42,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:14:44,860 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:46,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:48,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:50,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:14:54,302 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:14:56,016 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:14:56,017 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:15:02,847 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:15:02,848 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:15:02,848 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:15:02,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:15:04,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:06,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:08,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:10,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:11,148 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:15:11,149 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:15:22,885 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:15:22,886 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:15:22,886 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:15:23,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:15:24,375 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:15:24,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:26,280 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:15:26,280 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:15:26,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:28,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:30,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:41,410 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:15:41,411 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:15:42,923 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:15:42,923 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:15:42,924 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:15:43,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:15:44,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:46,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:48,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:50,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:15:54,448 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:15:56,542 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:15:56,542 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:16:02,960 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:16:02,960 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:16:02,961 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:16:03,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:16:04,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:06,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:08,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:10,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:11,671 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:16:11,671 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:16:22,999 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:16:23,000 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:16:23,000 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:16:23,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:16:24,518 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:16:24,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:26,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:16:26,801 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:16:26,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:28,905 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:30,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:41,932 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:16:41,933 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:16:43,047 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:16:43,047 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:16:43,048 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:16:43,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:16:44,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:46,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:48,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:50,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:16:54,590 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:16:57,063 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:16:57,063 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:17:03,088 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:17:03,088 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:17:03,088 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:17:03,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:17:04,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:06,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:08,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:10,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:12,194 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:17:12,195 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:17:23,123 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:17:23,124 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:17:23,124 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:17:23,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:17:24,664 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:17:24,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:26,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:27,326 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:17:27,327 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:17:28,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:30,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:42,457 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:17:42,457 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:17:43,168 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:17:43,168 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:17:43,168 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:17:43,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:17:44,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:46,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:48,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:50,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:17:54,739 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:17:57,588 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:17:57,588 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:18:03,206 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:18:03,206 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:18:03,207 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:18:03,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:18:04,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:06,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:08,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:10,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:12,722 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:18:12,723 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:18:23,245 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:18:23,246 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:18:23,246 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:18:23,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:18:24,813 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:18:24,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:26,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:27,855 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:18:27,855 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:18:28,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:30,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:42,987 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:18:42,987 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:18:43,283 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:18:43,283 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:18:43,284 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:18:43,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:18:44,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:46,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:48,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:50,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:18:54,887 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:18:58,122 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:18:58,122 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:19:03,320 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:19:03,321 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:19:03,321 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:19:03,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:19:04,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:06,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:08,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:10,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:13,254 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:19:13,254 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:19:23,359 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:19:23,359 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:19:23,360 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:19:23,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:19:24,961 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:19:24,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:26,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:28,384 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:19:28,384 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:19:28,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:31,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:33,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:35,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:37,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:39,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:41,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:43,530 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:19:43,530 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:19:43,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:45,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:47,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:49,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:51,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:53,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:55,030 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:19:55,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:57,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:19:58,662 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:19:58,662 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:19:59,996 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:01,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:03,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:05,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:08,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:10,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:12,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:13,792 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:20:13,793 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:20:14,003 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:16,004 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:18,005 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:20,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:22,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:24,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:25,101 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:20:27,194 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:20:27,194 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:20:27,194 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:20:28,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:28,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:20:28,942 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:20:28,943 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:20:30,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:32,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:34,012 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:36,013 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:44,076 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:20:44,077 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:20:47,243 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:20:47,243 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:20:47,244 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:20:48,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:48,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:20:50,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:52,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:54,021 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:55,176 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:20:56,022 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:20:59,207 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:20:59,208 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:21:07,281 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:21:07,281 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:21:07,282 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:21:08,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:08,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:21:10,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:12,030 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:14,031 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:14,340 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:21:14,340 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:21:16,032 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:25,249 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:21:27,318 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:21:27,318 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:21:27,318 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:21:28,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:28,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:21:29,468 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:21:29,469 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:21:30,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:32,039 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:34,040 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:36,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:44,599 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:21:44,599 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:21:47,355 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:21:47,356 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:21:47,356 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:21:48,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:48,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:21:50,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:52,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:54,048 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:55,324 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:21:56,049 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:21:59,731 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:21:59,731 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:22:07,392 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:22:07,393 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:22:07,393 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:22:08,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:08,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:22:10,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:12,056 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:14,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:14,860 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:22:14,861 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:22:16,058 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:25,401 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:22:27,430 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:22:27,430 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:22:27,430 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:22:28,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:28,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:22:30,027 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:22:30,028 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:22:30,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:32,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:34,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:36,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:45,160 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:22:45,160 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:22:47,476 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:22:47,477 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:22:47,477 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:22:48,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:48,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:22:50,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:52,073 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:54,073 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:22:55,478 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:22:57,075 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:00,291 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:23:00,291 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:23:07,514 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:23:07,515 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:23:07,515 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:23:08,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:23:09,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:10,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:11,082 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:13,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:15,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:15,429 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:23:15,429 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:23:17,084 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:25,548 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:23:27,552 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:23:27,553 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:23:27,553 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:23:28,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:23:29,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:30,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:30,559 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:23:30,559 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:23:31,091 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:33,091 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:35,092 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:37,094 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:45,694 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:23:45,695 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:23:47,589 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:23:47,589 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:23:47,589 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:23:48,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:23:49,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:50,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:51,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:53,100 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:55,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:23:55,624 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:23:57,102 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:00,832 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:24:00,833 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:24:07,626 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:24:07,627 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:24:07,627 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:24:08,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:24:09,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:10,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:11,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:13,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:15,110 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:15,963 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:24:15,963 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:24:17,111 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:25,697 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:24:27,662 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:24:27,662 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:24:27,662 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:24:28,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:28,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:24:29,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:31,094 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:24:31,095 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:24:31,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:33,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:35,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:37,119 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:46,224 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:24:46,225 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:24:47,699 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:24:47,700 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:24:47,700 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:24:48,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:48,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:24:49,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:51,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:53,126 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:55,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:24:55,771 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:24:57,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:01,355 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:25:01,355 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:25:07,739 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:25:07,739 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:25:07,740 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:25:08,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:08,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:25:09,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:11,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:13,134 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:15,135 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:16,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:25:16,488 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:25:17,136 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:25,845 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:25:27,779 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:25:27,780 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:25:27,780 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:25:28,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:25:29,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:31,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:31,619 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:25:31,619 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:25:33,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:35,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:37,144 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:46,751 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:25:46,752 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:25:47,818 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:25:47,818 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:25:47,819 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:25:48,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:25:49,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:51,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:53,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:55,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:25:55,917 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:25:57,152 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:01,882 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:26:01,883 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:26:07,859 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:26:07,860 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:26:07,860 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:26:08,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:26:09,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:11,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:13,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:15,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:17,024 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:26:17,025 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:26:17,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:25,987 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:26:27,899 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:26:27,899 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:26:27,900 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:26:28,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:26:29,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:31,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:32,157 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:26:32,157 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:26:33,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:35,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:37,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:47,289 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:26:47,290 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:26:47,936 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:26:47,936 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:26:47,937 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:26:48,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:26:49,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:51,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:53,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:55,176 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:26:56,059 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:26:57,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:02,420 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:27:02,421 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:27:07,975 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:27:07,976 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:27:07,976 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:27:08,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:27:09,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:11,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:13,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:15,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:17,185 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:17,552 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:27:17,552 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:27:26,133 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:27:28,024 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:27:28,024 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:27:28,025 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:27:28,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:27:29,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:31,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:32,695 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:27:32,695 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:27:33,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:35,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:37,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:47,826 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:27:47,827 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:27:48,066 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:27:48,066 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:27:48,067 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:27:48,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:27:49,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:51,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:53,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:55,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:27:56,211 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:27:57,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:02,957 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:28:02,958 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:28:08,107 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:28:08,108 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:28:08,108 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:28:08,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:28:09,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:11,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:13,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:15,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:17,212 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:18,103 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:28:18,104 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:28:26,284 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:28:28,145 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:28:28,145 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:28:28,146 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:28:28,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:28:29,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:31,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:33,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:33,235 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:28:33,235 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:28:35,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:37,220 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:48,190 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:28:48,190 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:28:48,191 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:28:48,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:28:48,378 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:28:48,379 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:28:49,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:51,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:53,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:55,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:28:56,361 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:28:57,228 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:03,513 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:29:03,513 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:29:08,237 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:29:08,237 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:29:08,238 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:29:09,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:09,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:29:11,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:13,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:15,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:17,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:18,645 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:29:18,645 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:29:26,433 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:29:28,284 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:29:28,284 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:29:28,285 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:29:29,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:29,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:29:31,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:33,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:33,778 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:29:33,778 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:29:35,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:37,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:48,323 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:29:48,323 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:29:48,323 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:29:48,937 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:29:48,938 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:29:49,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:49,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:29:50,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:52,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:54,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:56,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:29:56,509 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:29:58,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:04,079 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:30:04,079 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:30:08,362 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:30:08,362 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:30:08,363 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:30:09,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:09,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:30:10,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:12,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:14,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:16,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:18,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:19,211 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:30:19,212 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:30:26,583 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:30:28,401 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:30:28,401 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:30:28,402 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:30:29,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:29,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:30:30,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:32,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:34,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:34,346 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:30:34,346 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:30:36,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:48,442 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:30:48,443 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:30:48,443 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:30:49,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:49,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:30:49,511 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:30:49,511 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:30:50,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:52,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:54,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:56,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:30:56,657 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:30:58,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:04,643 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:31:04,644 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:31:08,480 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:31:08,480 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:31:08,481 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:31:09,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:09,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:31:10,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:12,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:14,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:16,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:18,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:19,776 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:31:19,776 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:31:26,730 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:31:28,518 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:31:28,518 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:31:28,519 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:31:29,292 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:29,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:31:30,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:32,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:34,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:34,916 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:31:34,916 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:31:36,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:38,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:48,558 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:31:48,559 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:31:48,559 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:31:49,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:49,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:31:50,060 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:31:50,061 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:31:50,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:52,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:54,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:56,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:31:56,803 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:31:58,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:05,211 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:32:05,211 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:32:08,597 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:32:08,598 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:32:08,598 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:32:09,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:09,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:32:10,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:12,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:14,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:16,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:18,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:20,345 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:32:20,346 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:32:26,876 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:32:28,639 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:32:28,639 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:32:28,640 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:32:29,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:29,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:32:30,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:32,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:34,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:35,517 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:32:35,518 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:32:36,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:48,672 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:32:48,672 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:32:48,673 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:32:49,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:49,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:32:50,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:50,653 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:32:50,653 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:32:52,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:54,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:56,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:32:56,952 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:33:05,793 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:33:05,794 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:33:08,710 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:33:08,710 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:33:08,711 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:33:09,336 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:09,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:33:10,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:12,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:14,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:16,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:20,924 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:33:20,924 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:33:27,027 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:33:28,749 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:33:28,749 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:33:28,750 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:33:29,344 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:29,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:33:30,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:32,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:34,347 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:36,065 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:33:36,066 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:33:36,347 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:48,797 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:33:48,797 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:33:48,798 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:33:49,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:49,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:33:50,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:51,201 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:33:51,201 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:33:52,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:54,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:56,355 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:33:57,099 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:34:06,336 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:34:06,336 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:34:08,834 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:34:08,834 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:34:08,835 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:34:09,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:34:10,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:12,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:14,362 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:16,363 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:21,469 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:34:21,470 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:34:27,172 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:34:28,875 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:34:28,875 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:34:28,876 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:34:29,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:34:30,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:32,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:34,370 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:36,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:36,602 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:34:36,603 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:34:48,940 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:34:48,940 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:34:48,940 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:34:49,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:34:50,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:51,740 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:34:51,740 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:34:52,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:54,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:56,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:34:57,248 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:35:06,872 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:35:06,873 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:35:08,977 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:35:08,977 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:35:08,978 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:35:09,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:35:10,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:12,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:14,388 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:16,388 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:22,006 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:35:22,006 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:35:27,319 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:35:29,020 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:35:29,020 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:35:29,020 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:35:29,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:35:30,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:32,396 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:34,397 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:36,397 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:37,140 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:35:37,141 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:35:49,058 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:35:49,058 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:35:49,059 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:35:49,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:35:50,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:52,273 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:35:52,274 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:35:52,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:54,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:56,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:35:57,391 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:36:07,404 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:36:07,404 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:36:09,097 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:36:09,098 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:36:09,098 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:36:09,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:36:10,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:11,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:13,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:15,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:17,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:22,537 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:36:22,537 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:36:27,462 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:36:29,146 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:36:29,146 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:36:29,147 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:36:29,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:36:30,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:31,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:33,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:35,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:37,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:37,668 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:36:37,669 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:36:49,180 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:36:49,181 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:36:49,181 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:36:49,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:36:50,427 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:51,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:52,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:36:52,802 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:36:53,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:55,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:57,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:36:57,536 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:37:07,937 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:37:07,937 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:37:09,656 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:37:09,656 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:37:09,657 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:37:10,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:10,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:37:11,436 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:13,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:15,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:17,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:19,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:24,659 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:37:24,659 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:37:27,599 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:37:29,253 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:37:29,253 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:37:29,254 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:37:29,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:37:30,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:31,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:33,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:35,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:37,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:39,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:39,798 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:37:39,798 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:37:49,291 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:37:49,291 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:37:49,292 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:37:49,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:37:50,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:51,453 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:53,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:54,930 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:37:54,930 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:37:55,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:57,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:37:57,667 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:37:59,457 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:09,327 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:38:09,328 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:38:09,328 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:38:09,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:38:10,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:38:10,088 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:38:10,461 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:11,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:13,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:15,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:17,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:25,228 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:38:25,229 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:38:27,732 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:38:29,364 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:38:29,364 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:38:29,365 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:38:29,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:38:30,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:31,470 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:33,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:35,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:37,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:40,359 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:38:40,360 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:38:49,400 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:38:49,400 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:38:49,401 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:38:49,478 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:38:50,478 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:51,478 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:53,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:55,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:55,502 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:38:55,503 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:38:57,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:38:57,797 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:39:09,456 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:39:09,456 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:39:09,457 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:39:09,487 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:39:10,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:10,669 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:39:10,670 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:39:11,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:13,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:15,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:17,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:25,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:39:25,801 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:39:27,864 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:39:29,492 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:39:29,492 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:39:29,493 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:39:29,496 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:39:30,496 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:31,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:33,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:35,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:37,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:40,936 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:39:40,936 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:39:49,540 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:39:49,540 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:39:49,541 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:39:50,504 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:50,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:39:51,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:53,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:55,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:56,066 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:39:56,067 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:39:57,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:39:57,930 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:40:09,578 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:40:09,578 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:40:09,579 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:40:10,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:10,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:40:11,213 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:40:11,214 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:40:11,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:13,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:15,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:17,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:19,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:26,355 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:40:26,356 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:40:27,997 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:40:29,623 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:40:29,623 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:40:29,624 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:40:30,521 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:30,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:40:31,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:33,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:35,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:37,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:41,492 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:40:41,492 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:40:49,666 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:40:49,667 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:40:49,667 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:40:50,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:50,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:40:51,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:53,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:55,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:56,626 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:40:56,627 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:40:57,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:40:58,064 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:41:09,702 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:41:09,703 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:41:09,703 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:41:10,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:10,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:41:11,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:11,784 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:41:11,785 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:41:13,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:15,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:17,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:26,915 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:41:26,915 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:41:28,132 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:41:29,739 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:41:29,739 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:41:29,740 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:41:30,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:30,546 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:41:31,547 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:33,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:35,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:37,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:42,047 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:41:42,047 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:41:49,775 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:41:49,775 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:41:49,776 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:41:50,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:50,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:41:51,555 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:53,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:55,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:57,180 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:41:57,180 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:41:57,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:41:58,203 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:42:09,812 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:42:09,812 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:42:09,812 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:42:10,563 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:10,564 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:42:11,564 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:12,342 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:42:12,343 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:42:13,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:15,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:17,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:27,478 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:42:27,479 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:42:28,273 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:42:29,856 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:42:29,856 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:42:29,857 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:42:30,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:42:31,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:33,572 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:35,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:37,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:42,612 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:42:42,613 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:42:49,899 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:42:49,899 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:42:49,900 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:42:50,579 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:42:51,579 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:53,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:55,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:57,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:42:57,747 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:42:57,748 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:42:58,345 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:43:09,939 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:43:09,939 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:43:09,940 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:43:10,587 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:43:11,588 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:12,879 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:43:12,879 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:43:13,589 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:15,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:17,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:28,013 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:43:28,013 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:43:28,416 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:43:29,978 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:43:29,978 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:43:29,979 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:43:30,596 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:43:31,596 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:33,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:35,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:37,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:43,145 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:43:43,146 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:43:50,019 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:43:50,019 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:43:50,019 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:43:50,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:43:51,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:53,605 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:55,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:57,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:43:58,289 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:43:58,289 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:43:58,490 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:44:10,059 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:44:10,059 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:44:10,060 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:44:10,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:44:11,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:13,423 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:44:13,423 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:44:13,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:15,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:17,614 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:28,559 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:44:28,560 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:44:28,690 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:44:30,097 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:44:30,097 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:44:30,098 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:44:30,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:44:31,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:33,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:35,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:37,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:43,692 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:44:43,692 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:44:50,134 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:44:50,135 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:44:50,135 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:44:50,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:44:51,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:52,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:54,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:56,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:58,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:44:58,637 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:44:58,825 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:44:58,825 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:45:10,172 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:45:10,172 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:45:10,175 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:45:10,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:45:11,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:12,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:13,956 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:45:13,956 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:45:14,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:16,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:18,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:28,708 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:45:29,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:45:29,089 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:45:30,212 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:45:30,212 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:45:30,213 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:45:30,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:45:31,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:32,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:34,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:36,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:38,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:44,223 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:45:44,223 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:45:50,250 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:45:50,250 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:45:50,253 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:45:50,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:45:51,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:52,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:54,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:56,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:58,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:45:58,782 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:45:59,359 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:45:59,359 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:46:10,290 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:46:10,290 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:46:10,290 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:46:10,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:46:11,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:12,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:14,495 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:46:14,495 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:46:14,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:16,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:18,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:28,854 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:46:29,627 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:46:29,628 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:46:30,329 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:46:30,329 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:46:30,330 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:46:30,670 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:46:31,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:32,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:34,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:36,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:38,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:44,759 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:46:44,760 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:46:50,374 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:46:50,374 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:46:50,375 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:46:50,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:46:51,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:52,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:54,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:56,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:58,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:46:58,929 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:46:59,899 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:46:59,899 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:47:10,429 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:47:10,429 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:47:10,429 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:47:10,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:47:11,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:12,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:14,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:15,029 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:47:15,029 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:47:16,690 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:18,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:29,002 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:47:30,161 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:47:30,161 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:47:30,456 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:47:30,457 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:47:30,457 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:47:30,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:47:31,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:32,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:34,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:36,698 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:38,699 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:45,293 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:47:45,293 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:47:50,495 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:47:50,495 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:47:50,495 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:47:50,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:47:51,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:52,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:54,706 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:56,707 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:58,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:47:59,077 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:48:00,426 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:48:00,426 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:48:10,532 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:48:10,533 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:48:10,533 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:48:10,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:48:11,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:12,714 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:14,715 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:15,558 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:48:15,559 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:48:16,715 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:18,716 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:29,152 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:48:30,573 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:48:30,573 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:48:30,573 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:48:30,699 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:48:30,699 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:48:30,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:48:31,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:32,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:34,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:36,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:38,724 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:45,838 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:48:45,838 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:48:50,609 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:48:50,610 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:48:50,610 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:48:50,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:48:52,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:54,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:56,732 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:58,733 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:48:59,228 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:49:02,146 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:49:02,147 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:49:10,647 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:49:10,647 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:49:10,647 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:49:10,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:49:12,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:14,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:16,741 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:17,279 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:49:17,279 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:49:18,741 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:29,302 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:49:30,687 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:49:30,687 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:49:30,688 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:49:30,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:49:32,440 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:49:32,441 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:49:32,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:34,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:36,749 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:38,750 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:47,583 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:49:47,583 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:49:50,733 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:49:50,733 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:49:50,733 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:49:50,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:49:52,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:54,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:56,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:58,758 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:49:59,376 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:50:02,714 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:50:02,715 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:50:10,773 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:50:10,773 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:50:10,774 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:50:11,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:50:12,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:14,765 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:16,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:17,845 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:50:17,845 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:50:18,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:29,449 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:50:30,812 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:50:30,812 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:50:30,813 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:50:31,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:50:32,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:32,982 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:50:32,983 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:50:34,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:36,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:38,776 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:48,116 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:50:48,117 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:50:50,852 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:50:50,852 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:50:50,853 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:50:51,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:50:52,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:54,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:56,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:58,785 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:50:59,524 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:51:03,250 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:51:03,250 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:51:10,890 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:51:10,891 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:51:10,891 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:51:11,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:51:12,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:14,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:16,793 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:18,385 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:51:18,386 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:51:18,794 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:29,594 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:51:30,929 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:51:30,929 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:51:30,930 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:51:31,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:51:32,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:33,521 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:51:33,521 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:51:34,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:36,802 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:38,803 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:48,654 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:51:48,654 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:51:50,968 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:51:50,969 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:51:50,969 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:51:51,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:51:52,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:54,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:56,810 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:58,811 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:51:59,668 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:52:03,786 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:52:03,786 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:52:11,009 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:52:11,009 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:52:11,010 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:52:11,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:52:12,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:14,818 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:16,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:18,819 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:18,919 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:52:18,919 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:52:29,744 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:52:31,049 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:52:31,050 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:52:31,050 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:52:31,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:52:32,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:34,065 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:52:34,065 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:52:34,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:36,827 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:38,828 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:49,195 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:52:49,195 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:52:51,092 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:52:51,092 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:52:51,092 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:52:51,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:52:52,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:54,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:56,836 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:58,837 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:52:59,821 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:53:04,326 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:53:04,327 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:53:11,137 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:53:11,138 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:53:11,138 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:53:11,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:53:12,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:13,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:15,845 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:17,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:19,457 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:53:19,457 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:53:19,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:29,895 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:53:31,181 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:53:31,181 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:53:31,182 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:53:31,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:53:32,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:33,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:34,590 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:53:34,591 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:53:35,854 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:37,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:39,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:49,722 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:53:49,722 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:53:51,220 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:53:51,220 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:53:51,221 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:53:51,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:53:52,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:53,862 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:55,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:57,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:59,865 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:53:59,972 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:54:04,854 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:54:04,854 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:54:11,261 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:54:11,261 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:54:11,262 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:54:11,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:54:12,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:13,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:15,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:17,873 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:19,874 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:19,986 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:54:19,986 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:54:30,049 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:54:31,306 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:54:31,306 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:54:31,307 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:54:31,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:54:32,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:33,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:35,117 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:54:35,118 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:54:35,882 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:37,882 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:39,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:50,251 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:54:50,252 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:54:51,345 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:54:51,346 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:54:51,346 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:54:51,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:54:52,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:53,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:55,891 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:57,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:54:59,893 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:00,124 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:55:05,389 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:55:05,390 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:55:11,392 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:55:11,392 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:55:11,393 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:55:11,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:55:12,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:13,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:15,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:17,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:19,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:20,522 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:55:20,522 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:55:30,199 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:55:31,430 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:55:31,431 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:55:31,431 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:55:31,907 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:55:32,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:33,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:35,655 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:55:35,655 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:55:35,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:37,910 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:39,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:50,786 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:55:50,786 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:55:51,469 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:55:51,470 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:55:51,470 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:55:51,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:55:52,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:53,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:55,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:57,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:55:59,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:00,274 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:56:05,921 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:56:05,922 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:56:11,510 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:56:11,511 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:56:11,511 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:56:11,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:56:12,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:13,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:15,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:17,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:19,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:21,058 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:56:21,058 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:56:30,347 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:56:31,547 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:56:31,547 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:56:31,548 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:56:31,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:56:32,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:33,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:35,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:36,188 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:56:36,189 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:56:37,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:39,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:51,321 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:56:51,321 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:56:51,585 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:56:51,586 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:56:51,586 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:56:51,945 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:56:53,946 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:55,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:57,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:56:59,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:00,423 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:57:06,458 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:57:06,458 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:57:11,635 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:57:11,636 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:57:11,636 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:57:11,954 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:57:13,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:15,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:17,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:19,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:21,590 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:57:21,590 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:57:30,498 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:57:31,672 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:57:31,672 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:57:31,672 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:57:31,962 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:57:33,963 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:35,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:36,722 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:57:36,722 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:57:37,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:39,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:51,710 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:57:51,710 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:57:51,711 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:57:51,884 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:57:51,884 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:57:51,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:57:53,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:55,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:57,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:57:59,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:00,570 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:58:07,023 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:58:07,024 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:58:11,746 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:58:11,746 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:58:11,747 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:58:11,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:58:13,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:15,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:17,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:19,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:22,155 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:58:22,155 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:58:30,641 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:58:31,786 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:58:31,786 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:58:31,787 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:58:31,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:58:33,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:35,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:37,288 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:58:37,288 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:58:37,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:39,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:51,826 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:58:51,826 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:58:51,827 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:58:51,996 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:58:52,451 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:58:52,451 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:58:53,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:55,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:58:57,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:00,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:00,719 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:59:07,592 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:59:07,593 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:59:11,863 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:59:11,864 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:59:11,864 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:59:12,005 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:59:14,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:16,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:18,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:20,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:22,724 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:59:22,724 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:59:30,790 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 05:59:31,901 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:59:31,902 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:59:31,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:59:32,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:59:34,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:35,015 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:37,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:37,860 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:59:37,860 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:59:39,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:41,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:51,941 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 05:59:51,942 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 05:59:51,942 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 05:59:52,023 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 05:59:53,005 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 05:59:53,006 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 05:59:54,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:55,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:57,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 05:59:59,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:00,867 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:00:01,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:08,144 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:00:08,144 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:00:11,999 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:00:11,999 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:00:12,000 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:00:12,031 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:00:14,032 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:15,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:17,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:19,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:21,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:23,277 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:00:23,277 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:00:30,944 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:00:32,040 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:00:32,040 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:00:32,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:00:32,041 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:00:34,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:35,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:37,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:38,443 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:00:38,443 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:00:39,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:41,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:43,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:45,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:47,048 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:49,048 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:51,049 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:53,050 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:53,578 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:00:53,578 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:00:55,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:57,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:00:59,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:01,019 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:01:01,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:03,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:05,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:07,056 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:08,729 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:01:08,730 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:01:09,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:11,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:13,058 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:15,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:17,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:19,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:21,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:23,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:23,862 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:01:23,863 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:01:25,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:27,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:29,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:31,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:31,092 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:01:33,066 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:35,892 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:01:35,892 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:01:35,893 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:01:36,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:01:37,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:38,996 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:01:38,997 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:01:39,069 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:41,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:43,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:45,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:54,129 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:01:54,129 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:01:55,938 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:01:55,938 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:01:55,939 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:01:56,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:01:57,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:58,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:01:59,078 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:01,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:01,166 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:02:03,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:09,257 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:02:09,258 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:02:15,982 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:02:15,983 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:02:15,983 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:02:16,085 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:02:17,085 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:19,086 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:21,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:23,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:24,391 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:02:24,392 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:02:31,244 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:02:36,022 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:02:36,022 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:02:36,023 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:02:36,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:02:37,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:39,094 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:39,525 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:02:39,525 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:02:41,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:43,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:54,658 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:02:54,659 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:02:56,061 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:02:56,062 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:02:56,062 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:02:56,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:02:57,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:02:59,102 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:01,103 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:01,316 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:03:03,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:05,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:09,791 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:03:09,792 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:03:16,102 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:03:16,103 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:03:16,103 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:03:16,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:03:17,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:19,110 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:21,111 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:23,111 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:24,922 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:03:24,923 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:03:25,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:31,385 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:03:36,140 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:03:36,141 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:03:36,141 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:03:37,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:37,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:03:39,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:40,057 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:03:40,057 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:03:41,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:43,119 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:45,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:55,189 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:03:55,190 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:03:56,185 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:03:56,185 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:03:56,185 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:03:57,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:03:57,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:03:59,126 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:01,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:01,452 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:04:03,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:05,128 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:10,321 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:04:10,322 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:04:16,226 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:04:16,226 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:04:16,227 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:04:17,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:17,134 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:04:18,134 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:20,135 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:22,135 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:24,136 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:25,454 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:04:25,455 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:04:26,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:31,517 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:04:36,271 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:04:36,272 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:04:36,272 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:04:37,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:37,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:04:38,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:40,144 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:40,586 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:04:40,586 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:04:42,145 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:44,145 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:46,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:55,717 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:04:55,717 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:04:56,313 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:04:56,314 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:04:56,315 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:04:57,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:04:58,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:04:59,152 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:00,152 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:01,582 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:05:02,153 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:04,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:06,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:10,851 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:05:10,851 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:05:16,348 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:05:16,348 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:05:16,348 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:05:17,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:05:18,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:19,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:20,161 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:22,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:24,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:25,985 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:05:25,985 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:05:26,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:31,646 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:05:36,382 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:05:36,383 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:05:36,383 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:05:37,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:05:38,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:39,169 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:40,169 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:41,115 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:05:41,116 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:05:42,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:44,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:46,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:56,278 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:05:56,279 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:05:56,421 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:05:56,422 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:05:56,422 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:05:57,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:05:58,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:05:59,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:00,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:01,714 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:06:02,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:04,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:06,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:11,411 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:06:11,412 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:06:16,471 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:06:16,471 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:06:16,471 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:06:17,186 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:17,186 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:06:18,186 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:20,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:22,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:24,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:26,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:26,549 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:06:26,549 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:06:31,786 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:06:36,514 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:06:36,514 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:06:36,515 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:06:37,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:37,195 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:06:38,195 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:40,196 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:41,680 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:06:41,681 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:06:42,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:44,198 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:56,552 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:06:56,552 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:06:56,553 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:06:56,850 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:06:56,850 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:06:57,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:06:57,204 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:06:58,204 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:00,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:01,862 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:07:02,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:04,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:11,993 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:07:11,993 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:07:16,597 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:07:16,598 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:07:16,598 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:07:17,213 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:07:18,213 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:19,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:20,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:22,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:24,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:26,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:27,126 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:07:27,127 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:07:31,939 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:07:36,635 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:07:36,636 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:07:36,636 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:07:37,222 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:07:38,223 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:39,223 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:40,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:42,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:42,256 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:07:42,256 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:07:44,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:46,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:56,676 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:07:56,676 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:07:56,677 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:07:57,231 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:07:57,389 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:07:57,390 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:07:58,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:07:59,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:00,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:02,014 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:08:02,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:04,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:06,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:12,529 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:08:12,530 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:08:16,723 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:08:16,724 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:08:16,724 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:08:17,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:17,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:08:18,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:20,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:22,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:24,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:26,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:27,661 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:08:27,662 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:08:32,087 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:08:36,760 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:08:36,761 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:08:36,761 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:08:37,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:08:38,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:40,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:42,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:42,794 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:08:42,795 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:08:44,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:46,254 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:08:56,812 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:08:56,813 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:08:56,813 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:08:57,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:08:57,930 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:08:57,930 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:08:58,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:00,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:02,160 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:09:02,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:04,263 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:06,264 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:13,063 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:09:13,064 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:09:16,853 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:09:16,853 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:09:16,854 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:09:17,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:09:18,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:20,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:22,271 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:24,272 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:26,273 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:28,196 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:09:28,196 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:09:32,232 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:09:36,902 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:09:36,902 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:09:36,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:09:37,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:09:38,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:40,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:42,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:43,327 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:09:43,328 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:09:44,281 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:46,282 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:56,943 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:09:56,943 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:09:56,944 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:09:57,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:09:58,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:09:58,459 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:09:58,459 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:10:00,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:02,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:02,307 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:10:04,290 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:06,291 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:13,597 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:10:13,597 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:10:16,978 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:10:16,978 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:10:16,979 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:10:17,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:10:18,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:20,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:22,298 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:24,299 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:26,300 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:28,730 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:10:28,730 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:10:32,382 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:10:37,015 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:10:37,015 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:10:37,016 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:10:37,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:10:38,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:40,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:42,307 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:43,867 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:10:43,868 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:10:44,308 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:46,309 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:57,057 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:10:57,057 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:10:57,058 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:10:57,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:10:58,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:10:59,007 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:10:59,008 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:11:00,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:02,316 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:02,455 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:11:04,317 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:06,318 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:14,149 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:11:14,149 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:11:17,100 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:11:17,101 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:11:17,101 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:11:17,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:11:18,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:20,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:22,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:24,325 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:26,326 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:29,281 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:11:29,281 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:11:32,528 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:11:37,146 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:11:37,146 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:11:37,147 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:11:37,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:11:38,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:40,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:43,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:44,422 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:11:44,422 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:11:45,334 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:57,183 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:11:57,184 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:11:57,184 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:11:57,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:11:58,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:59,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:11:59,578 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:11:59,578 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:12:01,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:02,601 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:12:03,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:05,343 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:14,713 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:12:14,713 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:12:17,219 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:12:17,219 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:12:17,220 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:12:17,349 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:12:18,349 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:19,349 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:21,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:23,351 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:25,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:29,844 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:12:29,844 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:12:32,674 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:12:37,255 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:12:37,255 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:12:37,256 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:12:37,357 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:12:38,358 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:39,358 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:41,359 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:43,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:44,974 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:12:44,974 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:12:45,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:57,302 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:12:57,303 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:12:57,303 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:12:57,365 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:12:58,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:12:59,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:00,104 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:13:00,104 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:13:01,367 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:02,746 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:13:03,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:05,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:15,237 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:13:15,237 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:13:17,345 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:13:17,345 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:13:17,346 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:13:17,374 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:13:18,374 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:19,375 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:21,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:23,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:25,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:30,371 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:13:30,371 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:13:32,818 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:13:37,385 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:13:37,385 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:13:37,385 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:13:38,383 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:38,383 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:13:39,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:41,384 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:43,385 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:45,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:45,500 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:13:45,501 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:13:57,422 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:13:57,422 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:13:57,423 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:13:58,392 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:13:58,392 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:13:59,393 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:00,656 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:14:00,656 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:14:01,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:02,889 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:14:03,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:05,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:15,787 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:14:15,787 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:14:17,459 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:14:17,459 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:14:17,460 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:14:18,401 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:18,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:14:19,402 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:21,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:23,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:25,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:30,956 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:14:30,956 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:14:32,965 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:14:37,494 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:14:37,495 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:14:37,495 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:14:38,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:38,410 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:14:39,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:41,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:43,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:45,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:46,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:14:46,089 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:14:57,531 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:14:57,531 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:14:57,532 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:14:58,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:14:58,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:14:59,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:01,230 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:15:01,230 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:15:01,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:03,039 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:15:03,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:05,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:16,366 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:15:16,366 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:15:17,570 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:15:17,570 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:15:17,571 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:15:18,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:18,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:15:19,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:21,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:23,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:25,431 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:31,499 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:15:31,499 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:15:33,113 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:15:37,606 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:15:37,606 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:15:37,607 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:15:38,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:38,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:15:39,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:41,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:43,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:45,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:46,629 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:15:46,630 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:15:57,643 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:15:57,643 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:15:57,644 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:15:58,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:15:58,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:15:59,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:01,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:01,762 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:16:01,762 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:16:03,187 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:16:03,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:05,449 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:16,897 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:16:16,898 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:16:17,681 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:16:17,681 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:16:17,681 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:16:18,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:18,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:16:19,454 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:21,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:23,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:25,457 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:32,034 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:16:32,035 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:16:33,261 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:16:37,721 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:16:37,721 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:16:37,722 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:16:38,462 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:38,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:16:39,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:41,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:43,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:45,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:47,166 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:16:47,166 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:16:57,758 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:16:57,759 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:16:57,759 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:16:58,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:16:58,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:16:59,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:01,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:02,297 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:17:02,298 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:17:03,337 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:17:03,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:05,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:17,444 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:17:17,445 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:17:17,795 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:17:17,796 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:17:17,796 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:17:18,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:18,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:17:19,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:21,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:23,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:25,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:32,581 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:17:32,581 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:17:33,411 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:17:37,831 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:17:37,832 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:17:37,832 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:17:38,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:17:39,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:41,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:43,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:45,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:17:47,712 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:17:47,713 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:17:57,871 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:17:57,871 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:17:57,872 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:17:58,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:17:59,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:01,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:02,844 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:18:02,845 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:18:03,487 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:18:03,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:05,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:17,908 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:18:17,909 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:18:17,909 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:18:17,981 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:18:17,982 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:18:18,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:18:19,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:21,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:23,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:25,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:33,149 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:18:33,150 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:18:33,560 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:18:37,945 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:18:37,946 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:18:37,946 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:18:38,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:18:39,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:41,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:43,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:45,518 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:18:48,281 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:18:48,281 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:18:57,989 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:18:57,989 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:18:57,989 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:18:58,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:18:59,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:01,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:03,412 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:19:03,413 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:19:03,526 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:03,633 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:19:05,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:18,030 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:19:18,030 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:19:18,035 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:19:18,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:19:18,600 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:19:18,601 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:19:19,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:21,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:23,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:25,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:33,706 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:19:33,743 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:19:33,743 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:19:38,069 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:19:38,070 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:19:38,070 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:19:38,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:19:39,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:41,542 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:43,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:45,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:19:48,873 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:19:48,874 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:19:58,109 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:19:58,109 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:19:58,110 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:19:58,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:19:59,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:00,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:02,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:03,778 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:20:04,003 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:20:04,003 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:20:04,552 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:06,553 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:18,154 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:20:18,154 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:20:18,154 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:20:18,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:20:19,150 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:20:19,150 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:20:19,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:20,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:22,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:24,561 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:26,562 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:33,844 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:20:34,284 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:20:34,284 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:20:38,192 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:20:38,192 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:20:38,193 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:20:38,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:20:39,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:40,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:42,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:44,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:46,570 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:20:49,414 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:20:49,414 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:20:58,230 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:20:58,230 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:20:58,231 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:20:58,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:20:59,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:00,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:02,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:03,907 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:21:04,547 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:21:04,547 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:21:04,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:06,578 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:18,267 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:21:18,267 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:21:18,268 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:21:18,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:21:19,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:19,735 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:21:19,736 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:21:20,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:22,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:24,586 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:26,587 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:33,974 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:21:34,925 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:21:34,925 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:21:38,305 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:21:38,306 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:21:38,306 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:21:38,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:21:39,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:40,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:45,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:47,596 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:21:50,057 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:21:50,058 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:21:50,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log