diff --git "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" --- "a/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" +++ "b/wandb/run-20210715_185845-dq8uirtg/logs/debug-internal.log" @@ -1410,3 +1410,3828 @@ 2021-07-15 19:43:53,059 DEBUG SenderThread:735392 [sender.py:send():179] send: stats 2021-07-15 19:43:59,201 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status 2021-07-15 19:43:59,201 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:44:14,332 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:44:14,332 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:44:23,193 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:44:29,477 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:44:29,477 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:44:44,606 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:44:44,606 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:44:52,953 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:44:53,270 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:44:54,954 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:44:56,955 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:44:58,956 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:44:59,734 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:44:59,734 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:45:00,956 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:13,240 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:45:13,240 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:45:13,241 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:45:13,961 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:45:13,961 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:14,873 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:45:14,873 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:45:15,962 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:17,963 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:19,964 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:21,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:23,348 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:45:30,004 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:45:30,004 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:45:33,287 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:45:33,288 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:45:33,288 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:45:33,969 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:45:34,969 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:35,970 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:37,971 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:39,971 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:41,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:45,137 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:45:45,137 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:45:53,334 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:45:53,334 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:45:53,335 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:45:53,426 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:45:53,977 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:45:54,978 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:55,978 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:57,979 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:45:59,979 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:00,269 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:46:00,269 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:46:01,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:13,372 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:46:13,372 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:46:13,373 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:46:13,985 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:46:14,986 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:15,423 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:46:15,424 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:46:15,986 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:17,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:19,988 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:21,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:23,502 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:46:30,554 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:46:30,554 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:46:33,410 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:46:33,410 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:46:33,411 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:46:33,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:46:34,994 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:35,995 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:37,996 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:39,997 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:41,998 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:45,684 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:46:45,685 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:46:53,446 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:46:53,447 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:46:53,447 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:46:53,580 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:46:54,003 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:46:55,004 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:56,004 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:46:58,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:00,006 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:00,817 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:47:00,818 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:47:02,007 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:13,483 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:47:13,483 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:47:13,484 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:47:14,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:47:15,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:15,986 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:47:15,986 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:47:16,013 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:18,014 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:20,014 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:22,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:23,658 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:47:31,116 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:47:31,116 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:47:33,519 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:47:33,519 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:47:33,520 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:47:34,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:47:35,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:36,021 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:38,022 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:40,022 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:42,023 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:46,248 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:47:46,248 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:47:53,555 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:47:53,555 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:47:53,556 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:47:53,732 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:47:54,028 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:47:55,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:56,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:47:58,030 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:00,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:01,379 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:48:01,379 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:48:02,031 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:13,592 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:48:13,592 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:48:13,593 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:48:14,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:48:15,037 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:16,037 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:16,514 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:48:16,515 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:48:18,038 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:20,039 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:22,040 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:23,810 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:48:31,644 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:48:31,644 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:48:33,628 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:48:33,628 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:48:33,629 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:48:34,045 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:48:35,046 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:36,046 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:38,047 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:40,048 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:42,049 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:46,773 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:48:46,774 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:48:53,663 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:48:53,663 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:48:53,664 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:48:53,897 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:48:54,054 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:48:55,055 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:56,055 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:48:58,056 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:00,057 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:01,903 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:49:01,904 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:49:02,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:13,704 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:49:13,704 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:49:13,705 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:49:14,063 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:49:15,063 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:16,064 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:17,037 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:49:17,037 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:49:18,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:20,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:22,066 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:23,975 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:49:32,170 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:49:32,170 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:49:33,740 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:49:33,741 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:49:33,741 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:49:34,071 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:49:35,071 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:36,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:38,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:40,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:42,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:47,301 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:49:47,302 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:49:53,778 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:49:53,779 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:49:53,779 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:49:54,079 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:49:54,236 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:49:55,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:56,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:49:58,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:00,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:02,083 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:02,645 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:50:02,646 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:50:13,820 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:50:13,821 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:50:13,821 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:50:14,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:50:15,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:16,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:17,778 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:50:17,778 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:50:18,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:20,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:22,091 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:24,312 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:50:32,916 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:50:32,916 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:50:33,858 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:50:33,858 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:50:33,859 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:50:34,096 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:50:35,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:36,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:38,098 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:40,099 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:42,100 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:48,058 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:50:48,059 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:50:53,902 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:50:53,903 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:50:53,903 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:50:54,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:50:54,388 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:50:55,105 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:56,105 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:50:58,106 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:00,107 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:02,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:03,187 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:51:03,188 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:51:13,938 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:51:13,939 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:51:13,939 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:51:14,113 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:51:15,113 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:16,113 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:18,114 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:18,317 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:51:18,317 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:51:20,115 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:22,116 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:24,465 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:51:33,446 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:51:33,447 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:51:33,975 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:51:33,975 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:51:33,975 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:51:34,120 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:51:35,121 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:36,121 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:38,122 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:40,122 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:42,123 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:48,576 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:51:48,577 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:51:54,011 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:51:54,012 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:51:54,012 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:51:54,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:51:54,543 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:51:55,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:56,129 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:51:58,129 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:00,130 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:02,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:03,705 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:52:03,705 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:52:14,048 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:52:14,049 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:52:14,049 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:52:14,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:52:15,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:16,136 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:18,136 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:18,837 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:52:18,838 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:52:20,137 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:22,138 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:24,621 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:52:33,971 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:52:33,971 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:52:34,113 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:52:34,114 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:52:34,114 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:52:34,142 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:52:35,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:36,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:38,144 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:40,145 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:42,145 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:49,140 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:52:49,141 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:52:54,138 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:52:54,139 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:52:54,139 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:52:54,150 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:52:54,698 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:52:55,150 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:56,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:52:58,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:00,152 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:02,153 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:05,112 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:53:05,113 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:53:14,177 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:53:14,178 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:53:14,178 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:53:15,158 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:53:15,158 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:16,158 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:18,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:20,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:20,266 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:53:20,266 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:53:22,160 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:24,775 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:53:34,216 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:53:34,216 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:53:34,216 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:53:35,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:53:35,166 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:35,411 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:53:35,411 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:53:36,166 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:38,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:40,167 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:42,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:50,550 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:53:50,550 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:53:54,251 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:53:54,251 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:53:54,252 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:53:54,854 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:53:55,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:53:55,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:56,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:53:58,174 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:00,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:02,175 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:05,684 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:54:05,684 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:54:14,302 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:54:14,303 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:54:14,303 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:54:15,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:54:15,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:16,181 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:18,181 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:20,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:20,816 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:54:20,816 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:54:22,183 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:24,933 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:54:34,346 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:54:34,347 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:54:34,347 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:54:35,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:54:35,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:35,972 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:54:35,972 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:54:36,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:38,190 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:40,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:42,192 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:51,104 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:54:51,104 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:54:54,383 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:54:54,383 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:54:54,383 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:54:55,009 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:54:55,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:54:55,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:56,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:54:58,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:00,199 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:02,200 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:06,235 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:55:06,235 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:55:14,417 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:55:14,418 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:55:14,418 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:55:15,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:55:15,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:16,206 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:18,207 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:20,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:21,368 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:55:21,368 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:55:22,209 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:25,088 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:55:34,459 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:55:34,460 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:55:34,460 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:55:35,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:55:35,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:36,216 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:36,530 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:55:36,531 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:55:39,217 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:41,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:43,219 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:51,661 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:55:51,661 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:55:54,495 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:55:54,495 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:55:54,496 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:55:55,165 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:55:55,224 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:55:56,224 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:57,225 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:55:59,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:01,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:03,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:06,796 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:56:06,796 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:56:14,536 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:56:14,537 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:56:14,537 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:56:15,232 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:56:16,232 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:17,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:19,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:21,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:21,942 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:56:21,943 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:56:23,235 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:25,243 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:56:34,571 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:56:34,572 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:56:34,572 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:56:35,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:56:36,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:37,074 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:56:37,075 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:56:37,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:39,241 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:41,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:43,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:52,208 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:56:52,208 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:56:54,615 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:56:54,616 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:56:54,616 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:56:55,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:56:55,320 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:56:56,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:57,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:56:59,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:01,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:03,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:07,339 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:57:07,340 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:57:14,669 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:57:14,669 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:57:14,670 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:57:15,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:57:16,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:17,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:19,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:21,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:22,573 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:57:22,573 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:57:23,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:25,398 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:57:34,703 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:57:34,703 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:57:34,704 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:57:35,264 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:57:36,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:37,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:37,705 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:57:37,706 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:57:39,266 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:41,267 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:43,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:52,836 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:57:52,836 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:57:54,739 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:57:54,739 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:57:54,740 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:57:55,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:57:55,475 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:57:56,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:57,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:57:59,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:01,276 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:03,277 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:07,976 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:58:07,976 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:58:14,776 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:58:14,776 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:58:14,777 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:58:15,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:58:16,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:17,284 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:19,285 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:21,286 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:23,109 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:58:23,109 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:58:23,287 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:25,553 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:58:34,812 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:58:34,812 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:58:34,813 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:58:35,292 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:58:36,292 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:37,293 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:38,253 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:58:38,253 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:58:39,294 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:41,295 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:43,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:53,384 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:58:53,384 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:58:54,849 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:58:54,849 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:58:54,850 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:58:55,301 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:58:55,631 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:58:56,302 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:57,302 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:58:59,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:01,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:03,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:08,516 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:59:08,516 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:59:14,892 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:59:14,892 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:59:14,893 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:59:15,309 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:59:16,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:17,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:19,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:21,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:23,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:23,648 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:59:23,649 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:59:25,710 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:59:34,937 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:59:34,938 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:59:34,938 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:59:35,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:59:36,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:37,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:38,778 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:59:38,779 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:59:39,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:41,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:43,320 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:53,910 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 19:59:53,910 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 19:59:54,975 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 19:59:54,976 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 19:59:54,976 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 19:59:55,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 19:59:55,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:55,787 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 19:59:56,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:57,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 19:59:59,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:01,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:03,328 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:09,047 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:00:09,048 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:00:15,020 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:00:15,020 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:00:15,021 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:00:15,332 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:00:15,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:16,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:17,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:19,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:21,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:23,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:24,178 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:00:24,179 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:00:25,865 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:00:35,078 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:00:35,079 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:00:35,079 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:00:35,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:00:36,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:38,341 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:39,308 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:00:39,309 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:00:40,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:42,343 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:44,344 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:54,441 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:00:54,442 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:00:55,117 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:00:55,117 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:00:55,121 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:00:55,348 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:00:55,944 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:00:56,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:00:58,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:00,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:02,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:04,352 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:09,572 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:01:09,573 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:01:15,153 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:01:15,153 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:01:15,154 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:01:15,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:01:16,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:18,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:20,358 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:22,359 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:24,360 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:24,786 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:01:24,786 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:01:26,022 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:01:35,192 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:01:35,193 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:01:35,193 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:01:35,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:01:36,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:38,365 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:39,918 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:01:39,918 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:01:40,366 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:42,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:44,368 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:55,056 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:01:55,057 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:01:55,234 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:01:55,235 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:01:55,235 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:01:55,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:01:56,096 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:01:56,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:01:58,373 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:00,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:02,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:04,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:10,188 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:02:10,188 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:02:15,271 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:02:15,272 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:02:15,272 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:02:15,379 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:02:16,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:18,381 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:20,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:22,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:24,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:25,318 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:02:25,318 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:02:26,175 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:02:35,309 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:02:35,309 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:02:35,310 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:02:35,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:02:36,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:38,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:40,390 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:40,457 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:02:40,458 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:02:42,391 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:44,391 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:55,344 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:02:55,344 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:02:55,344 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:02:55,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:02:55,599 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:02:55,599 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:02:56,246 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:02:56,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:02:58,397 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:00,398 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:02,399 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:04,400 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:10,731 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:03:10,732 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:03:15,382 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:03:15,382 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:03:15,383 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:03:15,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:03:16,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:18,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:20,406 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:22,407 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:24,408 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:25,862 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:03:25,862 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:03:26,316 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:03:35,427 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:03:35,427 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:03:35,427 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:03:36,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:03:36,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:38,414 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:40,415 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:40,993 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:03:40,994 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:03:42,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:44,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:55,465 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:03:55,466 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:03:55,466 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:03:56,145 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:03:56,145 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:03:56,385 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:03:56,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:03:56,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:03:58,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:00,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:02,424 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:04,425 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:11,392 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:04:11,393 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:04:15,502 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:04:15,503 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:04:15,503 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:04:16,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:04:16,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:17,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:18,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:20,432 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:22,432 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:24,433 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:26,455 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:04:26,525 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:04:26,525 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:04:35,540 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:04:35,540 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:04:35,541 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:04:36,438 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:04:37,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:38,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:40,440 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:41,654 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:04:41,654 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:04:42,441 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:44,442 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:55,575 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:04:55,576 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:04:55,576 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:04:56,447 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:04:56,526 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:04:56,806 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:04:56,806 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:04:57,447 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:04:58,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:00,448 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:02,449 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:04,450 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:06,451 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:08,452 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:10,453 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:11,938 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:05:11,939 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:05:12,454 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:14,455 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:16,456 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:18,457 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:20,457 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:22,458 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:24,459 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:26,460 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:26,599 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:05:27,078 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:05:27,079 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:05:28,461 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:30,462 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:32,463 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:34,463 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:36,464 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:38,465 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:40,466 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:42,207 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:05:42,208 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:05:42,467 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:44,468 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:46,469 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:48,469 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:50,470 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:53,472 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:55,473 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:56,673 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:05:57,373 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:05:57,374 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:05:57,474 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:05:59,468 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:05:59,469 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:05:59,469 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:05:59,475 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:06:01,476 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:03,477 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:05,478 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:07,479 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:09,480 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:12,503 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:06:12,503 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:06:19,517 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:06:19,517 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:06:19,518 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:06:20,484 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:06:21,485 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:23,486 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:25,487 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:26,752 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:06:27,488 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:27,635 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:06:27,635 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:06:29,489 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:39,557 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:06:39,558 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:06:39,558 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:06:40,493 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:06:41,494 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:42,764 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:06:42,764 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:06:43,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:45,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:47,496 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:49,497 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:06:56,829 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:06:57,901 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:06:57,901 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:06:59,602 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:06:59,603 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:06:59,603 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:07:00,501 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:07:01,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:03,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:05,503 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:07,504 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:09,505 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:13,035 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:07:13,036 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:07:19,640 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:07:19,641 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:07:19,641 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:07:20,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:07:21,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:23,510 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:25,511 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:26,907 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:07:27,511 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:28,170 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:07:28,170 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:07:39,676 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:07:39,676 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:07:39,677 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:07:40,516 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:07:41,516 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:43,325 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:07:43,325 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:07:43,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:45,518 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:47,519 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:07:56,985 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:07:58,457 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:07:58,457 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:07:59,711 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:07:59,711 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:07:59,711 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:08:00,524 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:08:01,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:03,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:05,526 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:07,527 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:13,590 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:08:13,591 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:08:19,746 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:08:19,747 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:08:19,747 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:08:20,532 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:08:21,532 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:23,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:25,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:27,063 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:08:27,534 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:28,723 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:08:28,723 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:08:39,781 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:08:39,781 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:08:39,781 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:08:40,539 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:08:41,540 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:43,540 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:43,878 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:08:43,879 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:08:45,541 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:47,542 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:49,542 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:08:57,140 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:08:59,008 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:08:59,009 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:08:59,818 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:08:59,818 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:08:59,819 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:09:00,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:09:01,547 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:03,548 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:05,548 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:07,549 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:09,550 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:14,140 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:09:14,141 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:09:19,857 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:09:19,857 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:09:19,858 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:09:20,554 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:09:21,554 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:23,555 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:25,556 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:27,218 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:09:27,557 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:29,271 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:09:29,271 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:09:29,557 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:39,895 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:09:39,896 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:09:39,896 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:09:40,562 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:09:41,562 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:43,563 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:44,413 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:09:44,414 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:09:45,564 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:47,565 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:49,565 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:09:57,296 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:09:59,559 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:09:59,559 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:09:59,932 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:09:59,932 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:09:59,933 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:10:00,570 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:10:01,570 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:03,571 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:05,572 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:07,573 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:09,573 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:14,690 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:10:14,691 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:10:19,968 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:10:19,968 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:10:19,969 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:10:20,578 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:10:21,578 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:22,579 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:23,579 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:25,580 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:27,374 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:10:27,581 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:29,823 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:10:29,824 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:10:40,003 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:10:40,003 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:10:40,004 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:10:40,586 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:10:41,587 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:42,587 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:43,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:44,954 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:10:44,954 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:10:45,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:47,589 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:10:57,452 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:11:00,038 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:11:00,039 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:11:00,039 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:11:00,096 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:11:00,096 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:11:00,595 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:11:01,595 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:02,595 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:03,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:05,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:07,597 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:15,240 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:11:15,240 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:11:20,083 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:11:20,084 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:11:20,084 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:11:20,602 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:11:21,603 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:22,603 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:23,603 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:25,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:27,529 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:11:27,605 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:30,372 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:11:30,372 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:11:40,119 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:11:40,120 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:11:40,120 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:11:40,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:11:41,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:42,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:43,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:45,501 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:11:45,502 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:11:45,613 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:47,614 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:11:57,607 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:12:00,156 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:12:00,156 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:12:00,157 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:12:00,619 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:12:00,746 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:12:00,746 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:12:01,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:02,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:03,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:05,621 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:07,622 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:09,623 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:15,910 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:12:15,910 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:12:20,192 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:12:20,192 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:12:20,193 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:12:20,627 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:12:21,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:22,628 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:23,629 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:25,630 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:27,631 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:27,685 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:12:29,631 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:31,044 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:12:31,044 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:12:40,230 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:12:40,231 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:12:40,231 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:12:40,636 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:12:41,636 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:42,637 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:43,637 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:45,638 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:46,179 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:12:46,179 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:12:47,639 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:49,640 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:12:57,763 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:13:00,265 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:13:00,265 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:13:00,266 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:13:00,644 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:13:01,321 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:13:01,322 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:13:01,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:02,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:03,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:05,646 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:07,647 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:09,648 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:16,464 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:13:16,465 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:13:20,302 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:13:20,303 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:13:20,303 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:13:20,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:13:21,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:22,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:23,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:25,654 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:27,655 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:27,841 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:13:29,656 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:31,734 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:13:31,734 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:13:40,338 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:13:40,338 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:13:40,339 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:13:40,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:13:41,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:42,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:43,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:45,662 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:46,864 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:13:46,865 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:13:47,663 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:13:57,920 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:14:00,376 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:14:00,377 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:14:00,377 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:14:00,668 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:14:01,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:02,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:03,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:04,390 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:14:04,391 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:14:05,671 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:07,671 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:19,523 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:14:19,524 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:14:20,414 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:14:20,415 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:14:20,415 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:14:20,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:14:21,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:22,677 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:23,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:25,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:27,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:28,000 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:14:34,662 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:14:34,663 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:14:40,456 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:14:40,456 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:14:40,456 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:14:40,684 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:14:41,684 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:42,684 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:43,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:45,685 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:47,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:49,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:14:49,796 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:14:49,796 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:14:58,079 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:15:00,493 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:15:00,493 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:15:00,494 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:15:00,691 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:15:01,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:02,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:03,692 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:04,928 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:15:04,928 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:15:05,693 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:07,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:09,695 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:20,061 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:15:20,061 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:15:20,534 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:15:20,535 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:15:20,535 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:15:20,699 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:15:21,699 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:22,699 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:23,700 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:25,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:27,701 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:28,157 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:15:29,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:35,193 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:15:35,194 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:15:40,575 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:15:40,576 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:15:40,576 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:15:40,706 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:15:42,707 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:44,708 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:46,709 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:48,710 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:50,323 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:15:50,324 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:15:50,711 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:15:58,234 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:16:00,619 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:16:00,620 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:16:00,620 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:16:00,715 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:16:02,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:04,716 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:05,459 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:16:05,459 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:16:06,717 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:08,718 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:20,592 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:16:20,592 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:16:20,723 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:16:20,723 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:16:20,724 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:16:20,724 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:16:22,725 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:24,726 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:26,727 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:28,312 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:16:28,727 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:35,725 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:16:35,725 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:16:40,702 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:16:40,703 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:16:40,703 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:16:40,732 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:16:42,733 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:44,734 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:46,735 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:48,736 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:16:50,858 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:16:50,859 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:16:58,390 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:17:00,741 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:17:00,741 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:17:00,742 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:17:00,742 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:17:01,742 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:17:02,742 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:04,743 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:05,996 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:17:05,996 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:17:06,744 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:08,745 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:20,778 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:17:20,779 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:17:20,779 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:17:21,148 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:17:21,149 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:17:21,750 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:17:22,750 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:24,751 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:26,752 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:28,467 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:17:28,752 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:30,753 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:36,288 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:17:36,289 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:17:40,815 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:17:40,816 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:17:40,816 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:17:41,757 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:17:42,758 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:44,758 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:46,759 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:48,760 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:50,761 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:17:51,421 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:17:51,422 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:17:58,545 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:18:00,854 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:18:00,854 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:18:00,855 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:18:01,765 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:18:02,765 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:04,766 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:06,557 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:18:06,557 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:18:06,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:08,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:10,768 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:20,904 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:18:20,904 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:18:20,905 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:18:21,699 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:18:21,700 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:18:21,772 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:18:22,773 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:24,774 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:26,774 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:28,624 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:18:28,775 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:30,776 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:36,838 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:18:36,838 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:18:40,937 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:18:40,938 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:18:40,938 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:18:41,780 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:18:42,780 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:44,781 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:46,782 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:48,783 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:50,783 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:18:51,969 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:18:51,970 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:18:58,695 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:19:00,978 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:19:00,978 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:19:00,979 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:19:01,787 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:19:02,788 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:04,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:06,789 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:07,102 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:19:07,102 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:19:08,790 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:21,021 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:19:21,021 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:19:21,022 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:19:21,795 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:19:22,242 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:19:22,242 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:19:22,795 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:24,796 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:26,797 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:28,772 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:19:28,798 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:30,799 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:37,374 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:19:37,374 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:19:41,067 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:19:41,067 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:19:41,068 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:19:41,803 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:19:42,803 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:44,804 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:46,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:48,806 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:50,807 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:19:52,504 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:19:52,505 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:19:58,849 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:20:01,111 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:20:01,111 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:20:01,112 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:20:01,811 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:20:02,811 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:03,812 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:04,812 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:06,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:07,638 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:20:07,638 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:20:08,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:10,815 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:21,150 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:20:21,151 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:20:21,151 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:20:21,819 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:20:22,774 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:20:22,775 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:20:22,819 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:23,820 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:24,820 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:26,821 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:28,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:28,926 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:20:30,823 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:37,909 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:20:37,909 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:20:41,201 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:20:41,202 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:20:41,202 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:20:41,827 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:20:42,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:43,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:44,829 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:46,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:48,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:20:53,042 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:20:53,042 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:20:59,016 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:21:01,277 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:21:01,277 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:21:01,278 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:21:01,835 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:21:02,836 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:03,836 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:04,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:06,838 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:08,221 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:21:08,221 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:21:08,838 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:21,352 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:21:21,352 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:21:21,353 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:21:21,843 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:21:22,843 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:23,359 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:21:23,359 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:21:23,844 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:24,844 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:26,845 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:28,846 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:29,113 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:21:38,491 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:21:38,492 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:21:41,391 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:21:41,391 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:21:41,392 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:21:41,851 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:21:42,851 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:43,852 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:44,852 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:46,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:48,854 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:50,855 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:21:53,623 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:21:53,624 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:21:59,190 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:22:01,506 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:22:01,507 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:22:01,507 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:22:01,859 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:22:02,859 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:03,860 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:04,860 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:06,861 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:08,753 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:22:08,753 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:22:08,861 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:10,862 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:21,562 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:22:21,563 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:22:21,563 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:22:21,867 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:22:22,867 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:23,867 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:23,899 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:22:23,899 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:22:24,868 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:26,869 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:28,869 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:29,269 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:22:39,043 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:22:39,043 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:22:41,598 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:22:41,599 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:22:41,599 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:22:41,874 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:22:42,874 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:43,875 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:44,875 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:46,876 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:48,877 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:22:54,175 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:22:54,175 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:22:59,348 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:23:01,635 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:23:01,635 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:23:01,636 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:23:01,882 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:23:02,882 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:03,883 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:04,883 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:06,884 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:08,884 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:09,313 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:23:09,313 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:23:21,680 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:23:21,681 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:23:21,681 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:23:21,889 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:23:22,890 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:23,890 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:24,443 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:23:24,444 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:23:24,890 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:26,891 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:28,892 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:29,425 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:23:39,584 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:23:39,584 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:23:41,719 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:23:41,719 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:23:41,719 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:23:41,897 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:23:42,897 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:43,898 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:44,898 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:46,899 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:48,899 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:50,900 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:23:54,716 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:23:54,716 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:23:59,503 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:24:01,765 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:24:01,765 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:24:01,766 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:24:01,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:24:02,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:03,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:04,906 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:06,906 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:08,907 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:09,849 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:24:09,849 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:24:10,908 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:21,823 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:24:21,823 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:24:21,824 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:24:21,912 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:24:22,912 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:23,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:24,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:24,988 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:24:24,988 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:24:26,914 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:28,915 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:29,580 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:24:30,915 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:40,128 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:24:40,128 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:24:41,901 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:24:41,902 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:24:41,902 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:24:41,919 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:24:42,919 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:43,920 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:45,921 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:47,922 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:49,922 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:51,923 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:24:55,265 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:24:55,265 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:24:59,658 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:25:01,965 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:25:01,966 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:25:01,966 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:25:02,927 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:25:03,928 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:05,928 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:07,929 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:09,930 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:10,399 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:25:10,399 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:25:22,003 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:25:22,003 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:25:22,004 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:25:22,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:25:23,935 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:25,532 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:25:25,533 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:25:25,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:27,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:29,735 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:25:29,937 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:40,665 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:25:40,665 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:25:42,050 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:25:42,050 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:25:42,051 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:25:42,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:25:43,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:45,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:47,944 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:49,945 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:25:55,798 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:25:55,798 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:25:59,813 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:26:02,084 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:26:02,085 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:26:02,085 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:26:02,949 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:26:03,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:05,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:07,951 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:09,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:10,927 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:26:10,927 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:26:22,128 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:26:22,128 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:26:22,128 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:26:22,956 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:26:23,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:25,957 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:26,060 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:26:26,060 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:26:27,958 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:29,890 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:26:29,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:31,960 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:41,195 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:26:41,196 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:26:42,178 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:26:42,179 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:26:42,179 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:26:42,963 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:26:43,964 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:45,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:47,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:49,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:51,967 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:26:56,325 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:26:56,326 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:26:59,969 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:27:02,285 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:27:02,285 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:27:02,285 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:27:02,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:27:03,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:05,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:07,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:09,976 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:11,456 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:27:11,457 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:27:11,977 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:22,324 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:27:22,324 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:27:22,325 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:27:22,981 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:27:23,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:24,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:25,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:26,587 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:27:26,587 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:27:27,983 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:29,984 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:30,047 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:27:31,985 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:41,719 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:27:41,719 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:27:42,362 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:27:42,363 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:27:42,363 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:27:42,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:27:43,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:44,990 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:45,990 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:47,991 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:49,992 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:51,992 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:27:56,849 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:27:56,849 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:28:00,119 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:28:02,397 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:28:02,398 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:28:02,398 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:28:02,996 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:28:03,997 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:04,997 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:05,998 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:07,998 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:09,999 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:11,979 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:28:11,980 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:28:22,435 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:28:22,435 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:28:22,436 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:28:23,004 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:28:24,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:25,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:26,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:27,114 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:28:27,114 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:28:28,006 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:30,007 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:30,189 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:28:42,244 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:28:42,244 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:28:42,478 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:28:42,478 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:28:42,478 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:28:43,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:28:44,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:45,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:46,013 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:48,014 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:50,015 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:28:57,379 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:28:57,379 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:29:00,261 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:29:02,554 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:29:02,554 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:29:02,554 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:29:03,019 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:29:04,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:05,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:06,021 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:08,021 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:10,022 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:12,023 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:12,511 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:29:12,511 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:29:22,594 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:29:22,594 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:29:22,595 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:29:23,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:29:24,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:25,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:26,028 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:27,641 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:29:27,641 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:29:28,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:30,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:30,333 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:29:32,030 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:42,728 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:29:42,728 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:29:42,729 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:29:42,785 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:29:42,785 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:29:43,034 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:29:44,035 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:45,035 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:46,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:48,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:50,037 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:52,038 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:29:57,941 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:29:57,941 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:30:00,409 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:30:02,765 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:30:02,765 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:30:02,766 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:30:03,042 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:30:04,042 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:05,043 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:06,043 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:08,044 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:10,045 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:13,073 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:30:13,074 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:30:22,880 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:30:22,880 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:30:22,880 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:30:23,050 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:30:24,050 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:25,050 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:26,051 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:28,052 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:28,206 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:30:28,207 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:30:30,052 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:30,488 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:30:42,918 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:30:42,918 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:30:42,918 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:30:43,057 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:30:43,372 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:30:43,373 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:30:44,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:45,058 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:46,059 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:48,059 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:50,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:52,061 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:30:58,502 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:30:58,503 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:31:00,565 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:31:02,952 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:31:02,952 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:31:02,953 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:31:03,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:31:04,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:05,065 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:06,066 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:08,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:10,067 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:12,068 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:13,658 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:31:13,658 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:31:22,990 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:31:22,990 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:31:22,991 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:31:23,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:31:24,072 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:25,073 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:26,073 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:28,074 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:28,789 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:31:28,789 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:31:30,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:30,643 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:31:32,075 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:43,028 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:31:43,029 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:31:43,029 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:31:43,079 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:31:43,947 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:31:43,948 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:31:44,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:45,080 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:46,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:48,081 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:50,082 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:52,083 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:31:59,090 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:31:59,090 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:32:00,722 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:32:03,076 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:32:03,076 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:32:03,077 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:32:03,087 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:32:04,087 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:05,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:06,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:08,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:10,089 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:12,090 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:14,226 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:32:14,227 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:32:23,129 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:32:23,130 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:32:23,130 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:32:24,094 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:32:24,094 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:25,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:26,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:28,096 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:29,362 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:32:29,362 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:32:30,097 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:30,798 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:32:33,098 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:43,167 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:32:43,167 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:32:43,168 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:32:44,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:32:44,632 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:32:44,633 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:32:45,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:47,103 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:49,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:51,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:53,105 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:32:59,771 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:32:59,772 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:33:00,876 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:33:03,203 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:33:03,204 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:33:03,204 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:33:04,108 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:33:05,109 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:07,110 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:09,111 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:11,111 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:13,112 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:14,909 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:33:14,909 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:33:23,244 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:33:23,244 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:33:23,245 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:33:24,116 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:33:25,116 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:27,117 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:29,118 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:30,040 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:33:30,041 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:33:30,954 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:33:31,119 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:33,119 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:43,281 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:33:43,281 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:33:43,282 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:33:44,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:33:45,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:45,218 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:33:45,219 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:33:47,125 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:49,126 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:51,126 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:33:53,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:00,353 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:34:00,354 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:34:01,032 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:34:03,318 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:34:03,318 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:34:03,318 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:34:04,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:34:05,131 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:07,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:09,133 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:11,134 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:13,134 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:15,484 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:34:15,484 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:34:23,357 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:34:23,357 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:34:23,358 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:34:24,138 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:34:25,139 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:27,139 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:29,140 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:30,618 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:34:30,619 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:34:31,111 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:34:31,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:33,142 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:43,408 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:34:43,408 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:34:43,408 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:34:44,145 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:34:45,146 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:45,772 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:34:45,772 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:34:47,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:49,147 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:51,148 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:34:53,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:00,907 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:35:00,908 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:35:01,188 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:35:03,445 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:35:03,445 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:35:03,445 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:35:04,153 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:35:05,154 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:07,154 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:09,155 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:11,156 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:13,157 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:16,038 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:35:16,039 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:35:23,517 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:35:23,517 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:35:23,518 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:35:24,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:35:25,161 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:27,162 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:29,162 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:31,163 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:31,172 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:35:31,172 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:35:31,303 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:35:33,164 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:43,557 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:35:43,557 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:35:43,558 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:35:44,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:35:45,168 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:46,306 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:35:46,306 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:35:47,169 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:49,170 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:51,171 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:35:53,172 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:01,342 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:36:01,481 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:36:01,481 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:36:03,594 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:36:03,595 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:36:03,595 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:36:04,176 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:36:05,176 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:07,177 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:09,178 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:11,178 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:13,179 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:16,611 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:36:16,612 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:36:23,701 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:36:23,702 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:36:23,702 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:36:24,183 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:36:25,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:26,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:27,184 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:29,185 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:31,186 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:31,421 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:36:31,751 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:36:31,751 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:36:33,187 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:43,752 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:36:43,752 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:36:43,753 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:36:44,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:36:45,191 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:46,192 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:46,886 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:36:46,886 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:36:47,192 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:49,193 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:51,194 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:36:53,195 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:01,498 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:37:02,036 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:37:02,036 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:37:03,788 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:37:03,788 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:37:03,789 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:37:04,199 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:37:05,200 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:06,200 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:07,201 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:09,201 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:11,202 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:17,168 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:37:17,169 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:37:23,823 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:37:23,824 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:37:23,824 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:37:24,207 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:37:25,207 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:26,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:27,208 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:29,209 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:31,210 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:31,576 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:37:32,299 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:37:32,299 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:37:43,896 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:37:43,897 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:37:43,897 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:37:44,214 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:37:45,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:46,215 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:47,216 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:47,431 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:37:47,431 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:37:49,217 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:37:51,217 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:01,653 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:38:02,595 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:38:02,596 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:38:03,934 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:38:03,934 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:38:03,935 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:38:04,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:38:05,222 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:06,223 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:07,223 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:09,224 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:11,225 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:17,750 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:38:17,751 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:38:23,970 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:38:23,971 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:38:23,971 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:38:24,229 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:38:25,230 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:26,230 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:27,231 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:29,231 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:31,232 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:31,732 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:38:32,880 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:38:32,880 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:38:33,233 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:44,006 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:38:44,006 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:38:44,007 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:38:44,237 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:38:45,237 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:46,238 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:47,238 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:48,014 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:38:48,015 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:38:49,239 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:51,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:38:53,240 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:01,808 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:39:03,147 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:39:03,147 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:39:04,045 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:39:04,045 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:39:04,046 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:39:04,244 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:39:05,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:06,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:07,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:09,246 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:11,247 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:13,248 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:18,277 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:39:18,278 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:39:24,087 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:39:24,087 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:39:24,088 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:39:24,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:39:25,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:26,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:27,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:29,253 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:31,254 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:31,886 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:39:33,255 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:33,407 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:39:33,408 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:39:44,124 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:39:44,124 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:39:44,125 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:39:44,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:39:45,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:46,260 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:47,261 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:48,957 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:39:48,957 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:39:49,261 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:52,263 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:39:54,263 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:01,965 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:40:04,090 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:40:04,090 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:40:04,217 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:40:04,217 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:40:04,218 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:40:04,267 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:40:06,268 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:08,269 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:10,269 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:12,270 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:19,230 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:40:19,231 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:40:24,225 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:40:24,226 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:40:24,226 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:40:24,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:40:26,276 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:28,277 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:30,277 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:32,044 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:40:32,278 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:34,364 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:40:34,364 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:40:44,268 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:40:44,268 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:40:44,269 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:40:44,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:40:46,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:48,284 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:49,494 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:40:49,495 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:40:50,285 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:40:52,286 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:02,120 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:41:04,326 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:41:04,327 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:41:04,327 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:41:04,634 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:41:04,634 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:41:05,291 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:41:06,292 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:08,293 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:10,293 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:12,294 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:19,771 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:41:19,771 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:41:24,421 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:41:24,422 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:41:24,422 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:41:25,300 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:41:26,301 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:28,301 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:30,302 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:32,201 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:41:32,303 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:34,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:34,905 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:41:34,905 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:41:44,469 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:41:44,470 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:41:44,470 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:41:45,308 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:41:46,309 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:48,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:50,035 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:41:50,035 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:41:50,311 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:52,312 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:41:54,313 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:02,281 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:42:04,506 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:42:04,507 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:42:04,507 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:42:05,177 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:42:05,178 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:42:05,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:42:06,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:08,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:10,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:12,321 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:14,322 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:20,314 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:42:20,314 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:42:24,550 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:42:24,551 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:42:24,551 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:42:25,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:42:26,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:28,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:30,328 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:32,329 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:32,359 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:42:35,458 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:42:35,459 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:42:44,621 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:42:44,621 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:42:44,622 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:42:45,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:42:46,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:48,335 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:50,336 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:42:50,590 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:42:50,590 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:42:52,337 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:02,434 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:43:04,670 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:43:04,670 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:43:04,671 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:43:05,341 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:43:06,065 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:43:06,066 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:43:06,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:08,343 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:10,344 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:12,344 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:14,345 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:21,248 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:43:21,248 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:43:24,709 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:43:24,709 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:43:24,710 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:43:25,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:43:26,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:28,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:30,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:32,352 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:32,503 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:43:34,353 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:36,382 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:43:36,382 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:43:44,747 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:43:44,748 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:43:44,748 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:43:45,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:43:46,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:48,358 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:50,359 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:51,512 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:43:51,513 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:43:52,359 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:43:54,360 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:02,574 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:44:04,784 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:44:04,784 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:44:04,785 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:44:05,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:44:06,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:06,650 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:44:06,651 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:44:08,365 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:10,366 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:12,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:21,801 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:44:21,801 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:44:24,821 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:44:24,822 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:44:24,822 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:44:25,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:44:26,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:28,373 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:30,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:32,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:32,645 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:44:34,376 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:36,931 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:44:36,931 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:44:44,859 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:44:44,859 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:44:44,860 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:44:45,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:44:46,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:48,381 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:50,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:52,061 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:44:52,062 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:44:52,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:44:54,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:02,721 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:45:04,908 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:45:04,909 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:45:04,909 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:45:05,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:45:06,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:07,198 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:45:07,198 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:45:08,390 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:10,390 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:12,391 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:14,392 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:22,336 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:45:22,337 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:45:24,946 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:45:24,946 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:45:24,947 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:45:25,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:45:26,397 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:27,397 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:28,398 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:30,398 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:32,399 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:32,795 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:45:37,480 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:45:37,480 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:45:44,990 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:45:44,991 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:45:44,991 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:45:45,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:45:46,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:47,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:48,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:50,406 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:52,407 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:45:52,611 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:45:52,611 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:46:02,868 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:46:05,026 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:46:05,027 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:46:05,027 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:46:05,412 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:46:06,412 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:07,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:07,743 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:46:07,743 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:46:08,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:10,414 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:12,415 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:14,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:16,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:18,417 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:20,418 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:22,419 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:22,883 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:46:22,883 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:46:24,419 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:26,420 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:28,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:30,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:32,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:32,944 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:46:34,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:37,425 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:38,045 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:46:38,046 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:46:39,425 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:41,426 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:43,427 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:45,428 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:47,429 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:49,429 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:51,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:53,176 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:46:53,177 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:46:53,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:55,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:57,432 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:46:59,433 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:01,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:03,022 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:47:03,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:05,435 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:07,436 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:08,311 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:47:08,311 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:47:08,866 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:47:08,866 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:47:08,866 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:47:09,436 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:47:11,437 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:13,438 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:15,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:17,439 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:23,444 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:47:23,444 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:47:28,910 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:47:28,911 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:47:28,911 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:47:29,444 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:47:29,444 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:31,445 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:33,100 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:47:33,445 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:35,446 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:37,447 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:38,576 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:47:38,576 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:47:48,975 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:47:48,975 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:47:48,976 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:47:49,452 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:47:49,452 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:51,452 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:53,453 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:53,708 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:47:53,708 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:47:55,454 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:47:57,455 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:03,177 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:48:09,015 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:48:09,016 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:48:09,016 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:48:09,057 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:48:09,057 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:48:09,460 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:48:11,461 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:13,462 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:15,462 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:17,463 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:24,194 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:48:24,194 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:48:29,057 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:48:29,057 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:48:29,058 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:48:29,467 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:48:31,468 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:33,255 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:48:33,469 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:35,470 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:37,471 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:39,325 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:48:39,325 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:48:49,121 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:48:49,122 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:48:49,122 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:48:49,475 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:48:51,476 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:53,477 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:54,456 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:48:54,456 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:48:55,478 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:48:57,479 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:03,333 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:49:09,180 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:49:09,180 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:49:09,181 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:49:09,484 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:49:09,590 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:49:09,591 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:49:11,484 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:13,485 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:15,486 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:17,487 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:24,720 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:49:24,720 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:49:29,217 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:49:29,218 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:49:29,218 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:49:29,492 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:49:29,492 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:31,493 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:33,412 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:49:33,493 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:35,494 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:37,495 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:39,850 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:49:39,850 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:49:49,257 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:49:49,257 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:49:49,258 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:49:49,500 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:49:51,501 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:53,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:54,981 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:49:54,982 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:49:55,502 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:49:57,503 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:03,489 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:50:09,303 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:50:09,304 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:50:09,304 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:50:09,508 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:50:10,120 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:50:10,121 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:50:11,509 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:12,510 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:14,510 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:16,511 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:18,512 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:25,257 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:50:25,258 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:50:29,340 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:50:29,340 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:50:29,340 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:50:29,517 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:50:32,518 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:33,568 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:50:34,519 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:36,520 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:38,521 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:40,388 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:50:40,388 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:50:49,418 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:50:49,418 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:50:49,419 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:50:49,525 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:50:52,526 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:54,527 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:55,524 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:50:55,524 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:50:56,528 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:50:58,529 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:03,644 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:51:09,473 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:51:09,474 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:51:09,474 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:51:09,533 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:51:10,534 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:10,657 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:51:10,657 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:51:12,535 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:14,535 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:16,536 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:18,538 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:25,797 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:51:25,798 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:51:29,510 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:51:29,511 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:51:29,511 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:51:29,542 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:51:32,543 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:33,723 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:51:34,544 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:36,545 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:38,546 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:40,928 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:51:40,929 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:51:49,566 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:51:49,567 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:51:49,567 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:51:50,550 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:51:52,551 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:54,552 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:56,067 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:51:56,067 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:51:56,552 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:51:58,553 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:03,798 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:52:09,605 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:52:09,605 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:52:09,606 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:52:10,558 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:52:10,558 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:11,202 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:52:11,202 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:52:12,559 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:14,560 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:16,561 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:18,562 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:26,366 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:52:26,366 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:52:29,660 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:52:29,660 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:52:29,661 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:52:30,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:52:30,567 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:32,568 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:33,873 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:52:34,569 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:36,570 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:38,570 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:41,497 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:52:41,498 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:52:49,698 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:52:49,698 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:52:49,699 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:52:50,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:52:50,575 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:52,576 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:54,577 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:56,578 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:52:56,625 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:52:56,626 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:52:58,579 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:03,948 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:53:09,742 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:53:09,743 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:53:09,743 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:53:10,584 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:53:10,584 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:11,776 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:53:11,776 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:53:12,585 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:14,586 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:16,587 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:18,588 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:26,909 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:53:26,910 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:53:29,782 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:53:29,782 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:53:29,783 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:53:30,593 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:53:32,594 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:34,024 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:53:34,595 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:36,595 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:38,596 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:42,044 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:53:42,044 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:53:49,820 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:53:49,820 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:53:49,821 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:53:50,601 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:53:52,602 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:54,603 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:56,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:53:57,180 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:53:57,181 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:53:58,604 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:04,101 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:54:09,893 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:54:09,894 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:54:09,894 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:54:10,609 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:54:12,333 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:54:12,334 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:54:12,610 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:14,611 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:16,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:18,612 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:27,477 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:54:27,477 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:54:29,935 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:54:29,935 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:54:29,935 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:54:30,617 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:54:32,618 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:34,179 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:54:34,619 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:36,620 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:38,621 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:42,607 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:54:42,607 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:54:49,973 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:54:49,973 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:54:49,974 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:54:50,625 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:54:50,626 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:52,626 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:54,627 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:56,629 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:54:57,740 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:54:57,740 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:54:58,630 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:04,256 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:55:10,012 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:55:10,012 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:55:10,012 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:55:10,634 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:55:10,634 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:12,635 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:12,886 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:55:12,887 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:55:14,636 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:16,637 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:18,638 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:28,017 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:55:28,018 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:55:30,075 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:55:30,076 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:55:30,076 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:55:30,642 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:55:30,642 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:32,643 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:34,334 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:55:34,644 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:36,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:38,645 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:43,149 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:55:43,149 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:55:50,115 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:55:50,115 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:55:50,116 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:55:50,650 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:55:50,650 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:52,651 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:54,651 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:56,652 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:55:58,282 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:55:58,282 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:55:58,653 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:04,412 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:56:10,163 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:56:10,163 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:56:10,164 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:56:10,658 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:56:10,658 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:12,659 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:13,413 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:56:13,413 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:56:14,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:16,660 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:18,661 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:28,542 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:56:28,543 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:56:30,204 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:56:30,205 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:56:30,205 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:56:30,666 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:56:32,667 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:34,489 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:56:34,668 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:36,669 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:38,670 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:43,672 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:56:43,672 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:56:50,245 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:56:50,246 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:56:50,246 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:56:50,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:56:52,678 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:54,679 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:56,680 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:58,681 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:56:58,801 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:56:58,801 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:57:04,566 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:57:10,287 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:57:10,288 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:57:10,288 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:57:10,686 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:57:12,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:13,687 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:13,932 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:57:13,932 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:57:15,688 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:17,689 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:19,690 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:29,064 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:57:29,064 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:57:30,361 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:57:30,361 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:57:30,361 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:57:30,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:57:30,694 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:31,695 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:33,696 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:34,643 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:57:35,696 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:37,697 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:39,698 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:44,196 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:57:44,196 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:57:50,416 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:57:50,416 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:57:50,417 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:57:50,702 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:57:51,703 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:53,703 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:55,704 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:57,705 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:57:59,326 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:57:59,326 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:57:59,706 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:04,721 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:58:10,458 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:58:10,459 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:58:10,459 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:58:10,711 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:58:11,711 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:13,712 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:14,459 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:58:14,459 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:58:15,713 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:17,714 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:19,715 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:29,599 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:58:29,599 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:58:30,496 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:58:30,496 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:58:30,497 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:58:30,719 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:58:31,719 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:33,720 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:34,800 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:58:35,721 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:37,721 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:39,722 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:44,728 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:58:44,729 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:58:50,532 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:58:50,533 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:58:50,533 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:58:50,727 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:58:51,727 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:53,728 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:55,729 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:57,730 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:59,730 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:58:59,859 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:58:59,860 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:59:04,879 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:59:10,569 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:59:10,569 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:59:10,570 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:59:10,735 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:59:11,736 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:13,736 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:15,000 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:59:15,001 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:59:15,737 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:17,738 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:19,739 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:30,130 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:59:30,130 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:59:30,608 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:59:30,608 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:59:30,608 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:59:30,743 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:59:31,743 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:33,744 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:34,956 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 20:59:35,745 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:37,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:39,746 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:45,260 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 20:59:45,261 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 20:59:50,650 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 20:59:50,650 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 20:59:50,650 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 20:59:50,750 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 20:59:51,751 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:53,751 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:55,752 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:57,753 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 20:59:59,754 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:00,393 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:00:00,393 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:00:05,033 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:00:10,691 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:00:10,692 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:00:10,692 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:00:10,758 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:00:11,758 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:13,759 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:15,524 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:00:15,525 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:00:15,760 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:17,760 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:19,761 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:30,655 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:00:30,655 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:00:30,786 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:00:30,786 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:00:30,787 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:00:31,766 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:00:31,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:33,767 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:35,111 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:00:35,768 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:37,769 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:39,770 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:45,797 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:00:45,798 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:00:50,775 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:00:50,775 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:00:50,776 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:00:50,776 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:00:51,776 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:53,777 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:55,778 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:57,779 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:00:59,780 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:00,936 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:01:00,937 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:01:05,190 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:01:10,808 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:01:10,809 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:01:10,809 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:01:11,785 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:01:11,785 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:13,786 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:15,787 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:16,067 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:01:16,067 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:01:17,787 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:19,788 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:30,862 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:01:30,863 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:01:30,863 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:01:31,276 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:01:31,277 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:01:31,793 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:01:31,794 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:33,794 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:35,268 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:01:35,795 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:37,796 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:39,797 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:46,420 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:01:46,420 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:01:50,902 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:01:50,902 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:01:50,903 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:01:51,802 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:01:51,802 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:53,802 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:55,803 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:01:58,804 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:00,805 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:01,553 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:02:01,553 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:02:05,345 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:02:10,963 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:02:10,963 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:02:10,964 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:02:11,810 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:02:11,810 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:12,810 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:14,811 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:16,684 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:02:16,685 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:02:16,812 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:18,813 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:20,814 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:31,012 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:02:31,012 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:02:31,013 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:02:31,818 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:02:31,819 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:31,844 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:02:31,844 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:02:32,819 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:34,820 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:35,423 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:02:36,821 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:38,822 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:40,823 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:46,980 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:02:46,981 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:02:51,048 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:02:51,048 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:02:51,049 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:02:51,827 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:02:51,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:52,828 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:54,829 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:56,830 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:02:58,831 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:00,832 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:02,113 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:03:02,114 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:03:05,498 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:03:11,106 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:03:11,106 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:03:11,107 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:03:11,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:03:11,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:12,837 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:14,838 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:16,839 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:17,248 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:03:17,248 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:03:18,840 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:20,840 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:31,142 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:03:31,142 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:03:31,143 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:03:31,844 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:03:31,845 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:32,382 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:03:32,382 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:03:32,845 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:34,846 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:35,576 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:03:36,847 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:38,848 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:47,518 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:03:47,518 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:03:51,183 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:03:51,184 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:03:51,184 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:03:51,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:03:51,853 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:52,854 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:54,854 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:56,855 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:03:58,856 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:02,651 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:04:02,651 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:04:05,655 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:04:11,252 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:04:11,253 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:04:11,253 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:04:11,861 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:04:11,862 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:12,862 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:14,863 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:16,864 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:17,783 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:04:17,784 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:04:18,864 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:20,865 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:31,290 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:04:31,291 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:04:31,291 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:04:31,870 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:04:31,870 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:32,870 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:32,930 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:04:32,930 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:04:34,871 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:35,733 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:04:36,872 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:38,873 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:40,873 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:48,063 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:04:48,063 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:04:51,328 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:04:51,328 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:04:51,329 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:04:51,878 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:04:52,878 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:54,879 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:56,880 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:04:58,881 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:00,881 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:03,198 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:05:03,199 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:05:05,811 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:05:11,365 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:05:11,366 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:05:11,366 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:05:11,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:05:12,886 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:14,887 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:16,888 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:18,331 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:05:18,331 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:05:18,888 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:20,889 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:31,417 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:05:31,418 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:05:31,418 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:05:31,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:05:32,894 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:33,496 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:05:33,496 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:05:34,895 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:35,888 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:05:36,896 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:38,897 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:40,898 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:48,636 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:05:48,637 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:05:51,452 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:05:51,453 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:05:51,453 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:05:51,902 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:05:52,903 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:54,904 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:56,904 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:05:58,905 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:03,766 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:06:03,766 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:06:05,962 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:06:11,488 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:06:11,488 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:06:11,489 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:06:11,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:06:12,910 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:14,911 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:16,912 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:18,899 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:06:18,899 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:06:18,913 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:31,531 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:06:31,531 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:06:31,532 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:06:31,917 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:06:32,918 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:34,038 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:06:34,038 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:06:34,918 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:36,030 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:06:36,919 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:38,920 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:49,170 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:06:49,170 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:06:51,566 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:06:51,567 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:06:51,567 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:06:51,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:06:52,925 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:54,926 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:56,927 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:06:58,928 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:04,302 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:07:04,303 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:07:06,098 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:07:11,612 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:07:11,613 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:07:11,613 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:07:11,932 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:07:12,933 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:14,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:16,934 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:18,935 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:19,434 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:07:19,434 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:07:20,936 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:31,687 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:07:31,687 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:07:31,688 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:07:31,940 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:07:32,941 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:34,570 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:07:34,571 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:07:34,941 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:36,172 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:07:36,942 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:38,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:40,943 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:49,701 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:07:49,702 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:07:51,724 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:07:51,724 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:07:51,725 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:07:51,948 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:07:52,948 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:54,949 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:56,949 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:07:58,950 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:00,952 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:04,831 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:08:04,832 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:08:06,247 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:08:11,760 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:08:11,760 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:08:11,760 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:08:11,955 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:08:12,956 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:14,956 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:17,958 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:19,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:20,056 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:08:20,056 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:08:21,959 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:23,960 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:31,796 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:08:31,797 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:08:31,797 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:08:31,963 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:08:32,964 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:33,964 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:35,277 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:08:35,277 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:08:35,965 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:36,319 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:08:37,966 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:39,967 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:41,968 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:43,968 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:50,477 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:08:50,478 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:08:51,986 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:08:51,986 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:08:51,987 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:08:52,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:08:53,972 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:55,973 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:57,974 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:08:59,975 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:05,614 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:09:05,614 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:09:06,393 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:09:12,148 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:09:12,148 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:09:12,149 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:09:12,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:09:13,980 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:15,981 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:17,982 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:19,983 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:20,747 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:09:20,748 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:09:32,185 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:09:32,185 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:09:32,186 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:09:32,987 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:09:33,988 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:35,882 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:09:35,882 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:09:35,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:36,467 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:09:37,989 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:39,990 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:41,991 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:51,015 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:09:51,016 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:09:52,223 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:09:52,223 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:09:52,224 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:09:52,995 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:09:53,996 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:55,997 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:57,997 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:09:59,998 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:01,999 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:06,145 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:10:06,145 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:10:06,541 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:10:12,261 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:10:12,262 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:10:12,262 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:10:13,003 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:10:14,004 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:16,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:18,005 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:20,006 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:21,274 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:10:21,275 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:10:32,297 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:10:32,298 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:10:32,298 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:10:33,011 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:10:34,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:36,012 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:36,412 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:10:36,413 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:10:36,613 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:10:38,013 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:40,014 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:51,546 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:10:51,546 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:10:52,334 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:10:52,335 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:10:52,335 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:10:53,019 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:10:54,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:56,020 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:10:58,021 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:00,022 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:06,679 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:11:06,680 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:11:06,814 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:11:12,370 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:11:12,370 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:11:12,371 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:11:13,027 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:11:14,028 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:16,028 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:18,029 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:20,030 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:21,816 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:11:21,816 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:11:32,405 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:11:32,406 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:11:32,406 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:11:33,035 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:11:34,035 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:36,036 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:36,765 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:11:36,945 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:11:36,946 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:11:38,037 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:40,038 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:52,078 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:11:52,078 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:11:52,453 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:11:52,453 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:11:52,454 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:11:53,043 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:11:54,044 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:56,045 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:11:58,045 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:00,046 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:06,843 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:12:07,210 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:12:07,211 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:12:12,505 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:12:12,505 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:12:12,508 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:12:13,052 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:12:14,052 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:16,053 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:18,054 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:20,055 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:22,342 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:12:22,342 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:12:32,553 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:12:32,554 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:12:32,554 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:12:33,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:12:34,060 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:36,061 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:36,921 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:12:37,476 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:12:37,476 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:12:38,062 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:40,063 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:52,590 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:12:52,590 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:12:52,593 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:12:52,670 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:12:52,670 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:12:53,068 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:12:54,068 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:56,069 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:12:58,070 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:00,071 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:07,000 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:13:07,825 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:13:07,826 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:13:12,670 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:13:12,671 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:13:12,671 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:13:13,076 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:13:14,076 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:16,077 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:18,078 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:20,079 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:22,957 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:13:22,957 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:13:32,706 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:13:32,706 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:13:32,708 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:13:33,084 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:13:34,084 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:35,085 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:37,077 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:13:37,086 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:38,092 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:13:38,092 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:13:39,086 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:41,088 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:52,742 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:13:52,742 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:13:52,742 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:13:53,092 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:13:53,252 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:13:53,253 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:13:54,093 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:55,093 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:57,094 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:13:59,095 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:01,096 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:07,154 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:14:08,389 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:14:08,390 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:14:12,796 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:14:12,796 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:14:12,797 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:14:13,101 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:14:14,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:15,102 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:17,103 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:19,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:21,104 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:23,520 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:14:23,521 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:14:32,830 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:14:32,831 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:14:32,831 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:14:33,109 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:14:34,110 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:35,110 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:37,111 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:37,232 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:14:38,652 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:14:38,652 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:14:39,112 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:41,112 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:52,881 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:14:52,882 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:14:52,882 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:14:53,117 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:14:53,781 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:14:53,786 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:14:54,117 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:55,118 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:57,119 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:14:59,119 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:01,120 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:07,311 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:15:08,926 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:15:08,926 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:15:12,943 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:15:12,944 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:15:12,944 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:15:13,124 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:15:14,125 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:15,125 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:17,126 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:19,127 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:21,128 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:24,057 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:15:24,057 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:15:33,008 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:15:33,009 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:15:33,009 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:15:33,132 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:15:34,133 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:35,133 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:37,134 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:37,388 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:15:39,135 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:39,188 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:15:39,188 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:15:41,136 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:53,058 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:15:53,058 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:15:53,059 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:15:53,140 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:15:54,140 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:54,322 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:15:54,322 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:15:55,141 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:57,142 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:15:59,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:01,143 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:03,144 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:07,467 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:16:09,454 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:16:09,454 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:16:13,097 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:16:13,097 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:16:13,098 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:16:13,148 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:16:14,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:15,149 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:17,150 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:19,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:21,151 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:24,588 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:16:24,588 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:16:33,134 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:16:33,135 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:16:33,135 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:16:33,156 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:16:34,156 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:35,157 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:37,157 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:37,544 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:16:39,158 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:39,720 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:16:39,721 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:16:41,159 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:53,175 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:16:53,175 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:16:53,176 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:16:54,163 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:16:54,871 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:16:54,871 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:16:55,164 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:57,165 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:16:59,166 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:01,166 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:07,623 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:17:10,016 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:17:10,016 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:17:13,222 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:17:13,222 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:17:13,223 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:17:14,171 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:17:15,171 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:17,172 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:19,173 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:21,174 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:25,145 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:17:25,146 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:17:33,262 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:17:33,262 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:17:33,263 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:17:34,179 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:17:35,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:37,180 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:37,701 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:17:39,181 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:40,277 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:17:40,278 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:17:41,182 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:53,315 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:17:53,315 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:17:53,316 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:17:54,187 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:17:55,188 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:55,434 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:17:55,434 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:17:57,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:17:59,189 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:01,190 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:07,781 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:18:10,581 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:18:10,582 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:18:13,375 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:18:13,376 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:18:13,376 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:18:14,196 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:18:15,196 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:17,197 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:19,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:21,198 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:25,717 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:18:25,718 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:18:33,411 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:18:33,411 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:18:33,412 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:18:34,203 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:18:35,203 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:37,204 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:37,858 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:18:39,205 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:40,848 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:18:40,848 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:18:41,205 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:53,460 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:18:53,460 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:18:53,461 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:18:54,210 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:18:55,210 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:55,980 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:18:55,980 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:18:57,211 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:18:59,212 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:01,213 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:07,938 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:19:11,133 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:19:11,134 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:19:13,494 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:19:13,494 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:19:13,495 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:19:14,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:19:15,218 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:17,219 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:19,220 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:21,221 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:26,264 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:19:26,264 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:19:33,532 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:19:33,533 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:19:33,533 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:19:34,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:19:35,226 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:37,227 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:38,015 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:19:39,228 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:41,229 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:41,396 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:19:41,396 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:19:53,568 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:19:53,568 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:19:53,568 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:19:54,234 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:19:55,235 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:56,531 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:19:56,531 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:19:57,235 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:19:59,236 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:01,237 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:08,095 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:20:11,674 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:20:11,675 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:20:13,612 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:20:13,612 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:20:13,613 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:20:14,242 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:20:15,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:17,243 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:19,244 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:21,245 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:26,807 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:20:26,807 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:20:33,650 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:20:33,651 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:20:33,651 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:20:34,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:20:35,249 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:37,250 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:38,172 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:20:39,251 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:41,252 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:41,938 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:20:41,938 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:20:53,699 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:20:53,699 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:20:53,699 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:20:54,256 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:20:55,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:57,068 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:20:57,069 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:20:57,257 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:20:59,258 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:01,259 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:08,251 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:21:12,198 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:21:12,198 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:21:13,735 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:21:13,736 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:21:13,736 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:21:14,264 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:21:15,264 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:17,265 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:19,266 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:21,266 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:27,326 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:21:27,327 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:21:33,780 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:21:33,780 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:21:33,781 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:21:34,272 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:21:35,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:36,273 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:38,274 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:38,332 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:21:40,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:42,275 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:42,456 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:21:42,456 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:21:53,837 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:21:53,837 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:21:53,838 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:21:54,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:21:55,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:56,280 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:21:57,590 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:21:57,591 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:21:58,281 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:00,282 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:02,283 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:08,409 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:22:12,725 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:22:12,725 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:22:13,875 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:22:13,875 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:22:13,876 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:22:14,287 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:22:15,288 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:16,288 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:18,289 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:20,290 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:22,291 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:27,858 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:22:27,858 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:22:33,912 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:22:33,913 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:22:33,913 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:22:34,295 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:22:35,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:36,296 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:38,297 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:38,486 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:22:40,298 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:42,299 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:43,174 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:22:43,174 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:22:53,949 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:22:53,949 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:22:53,950 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:22:54,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:22:55,304 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:56,305 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:22:58,305 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:22:58,306 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:22:58,308 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:00,309 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:02,310 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:08,563 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:23:13,473 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:23:13,473 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:23:13,992 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:23:13,992 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:23:13,992 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:23:14,315 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:23:15,316 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:16,316 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:18,317 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:20,318 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:22,319 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:28,605 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:23:28,605 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:23:34,029 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:23:34,029 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:23:34,030 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:23:34,323 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:23:35,324 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:36,324 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:38,325 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:38,642 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:23:40,326 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:42,327 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:43,738 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:23:43,738 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:23:54,094 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:23:54,095 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:23:54,095 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:23:54,331 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:23:55,331 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:56,332 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:58,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:23:58,877 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:23:58,877 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:24:00,333 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:02,334 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:08,718 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:24:14,019 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:24:14,019 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:24:14,149 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:24:14,149 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:24:14,149 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:24:14,339 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:24:15,339 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:16,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:18,340 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:20,341 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:22,342 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:29,181 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:24:29,181 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:24:34,180 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:24:34,180 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:24:34,180 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:24:34,347 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:24:35,348 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:36,348 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:38,349 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:38,787 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:24:40,350 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:42,351 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:44,316 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:24:44,317 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:24:54,215 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:24:54,215 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:24:54,216 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:24:54,355 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:24:55,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:56,356 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:58,357 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:24:59,446 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:24:59,447 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:25:00,358 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:02,359 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:08,859 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:25:14,271 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:25:14,271 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:25:14,271 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:25:14,363 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:25:14,607 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:25:14,607 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:25:15,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:16,364 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:18,365 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:20,366 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:22,367 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:29,762 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:25:29,762 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:25:34,317 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:25:34,318 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:25:34,318 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:25:34,371 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:25:35,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:36,372 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:38,373 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:38,933 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:25:40,374 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:42,375 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:44,902 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:25:44,903 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:25:54,360 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:25:54,360 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:25:54,361 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:25:54,379 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:25:55,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:56,380 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:25:58,381 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:00,051 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:26:00,052 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:26:00,382 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:02,383 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:09,010 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:26:14,403 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:26:14,403 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:26:14,403 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:26:15,190 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:26:15,191 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:26:15,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:26:16,388 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:18,389 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:20,390 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:22,391 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:30,324 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:26:30,325 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:26:34,443 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:26:34,444 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:26:34,444 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:26:35,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:26:36,396 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:38,397 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:39,087 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:26:40,398 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:42,399 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:45,461 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:26:45,462 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:26:54,494 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:26:54,495 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:26:54,495 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:26:55,404 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:26:56,405 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:26:58,406 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:00,407 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:00,624 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:27:00,624 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:27:02,407 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:09,163 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:27:14,547 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:27:14,548 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:27:14,548 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:27:15,412 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:27:15,785 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:27:15,785 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:27:16,413 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:18,414 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:21,414 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:23,415 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:25,416 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:27,417 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:29,418 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:30,924 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:27:30,924 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:27:31,419 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:33,420 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:35,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:37,421 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:39,238 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:27:39,422 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:41,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:43,423 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:45,424 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:46,056 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:27:46,056 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:27:47,425 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:49,426 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:51,426 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:53,427 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:55,428 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:57,429 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:27:59,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:01,186 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:28:01,187 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:28:01,430 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:03,431 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:05,432 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:07,433 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:09,313 DEBUG SenderThread:735392 [sender.py:send():179] send: stats +2021-07-15 21:28:09,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:11,434 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:13,435 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:15,436 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:16,321 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:28:16,322 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:28:18,383 DEBUG SenderThread:735392 [sender.py:send():179] send: history +2021-07-15 21:28:18,384 DEBUG SenderThread:735392 [sender.py:send():179] send: summary +2021-07-15 21:28:18,384 INFO SenderThread:735392 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-15 21:28:18,437 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/wandb-summary.json +2021-07-15 21:28:26,441 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:28,442 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:30,442 INFO Thread-8 :735392 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210715_185845-dq8uirtg/files/output.log +2021-07-15 21:28:31,455 DEBUG HandlerThread:735392 [handler.py:handle_request():124] handle_request: stop_status +2021-07-15 21:28:31,456 DEBUG SenderThread:735392 [sender.py:send_request():193] send_request: stop_status +2021-07-15 21:28:39,389 DEBUG SenderThread:735392 [sender.py:send():179] send: stats