diff --git "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" --- "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" +++ "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" @@ -19592,3 +19592,3701 @@ 2021-07-17 07:21:08,448 DEBUG SenderThread:799749 [sender.py:send():179] send: stats 2021-07-17 07:21:08,728 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status 2021-07-17 07:21:08,729 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:21:23,858 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:21:23,859 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:21:38,519 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:21:38,989 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:21:38,989 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:21:54,127 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:21:54,127 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:22:08,593 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:22:09,258 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:22:09,258 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:22:24,391 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:22:24,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:22:29,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:31,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:33,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:35,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:37,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:38,667 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:22:39,522 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:22:39,522 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:22:47,851 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:22:47,852 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:22:47,852 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:22:48,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:22:49,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:51,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:53,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:55,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:22:55,579 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:22:55,579 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:22:57,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:07,932 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:23:07,933 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:23:07,933 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:23:08,131 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:23:08,738 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:23:09,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:10,729 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:23:10,730 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:23:11,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:13,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:15,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:17,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:25,971 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:23:25,971 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:23:27,970 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:23:27,971 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:23:27,971 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:23:28,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:23:29,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:30,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:31,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:33,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:35,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:37,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:38,812 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:23:41,100 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:23:41,101 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:23:48,081 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:23:48,081 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:23:48,082 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:23:48,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:23:49,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:50,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:51,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:53,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:55,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:23:56,237 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:23:56,238 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:23:57,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:08,162 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:24:08,163 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:24:08,163 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:24:08,886 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:24:09,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:09,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:24:10,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:11,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:11,388 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:24:11,389 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:24:13,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:15,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:17,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:26,580 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:24:26,580 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:24:28,249 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:24:28,249 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:24:28,250 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:24:29,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:29,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:24:30,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:31,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:33,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:35,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:37,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:38,964 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:24:41,710 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:24:41,710 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:24:48,287 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:24:48,288 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:24:48,288 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:24:49,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:49,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:24:50,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:51,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:53,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:55,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:24:56,843 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:24:56,843 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:24:57,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:08,350 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:25:08,351 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:25:08,351 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:25:09,038 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:25:09,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:25:10,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:11,974 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:25:11,975 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:25:12,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:14,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:16,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:18,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:27,105 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:25:27,106 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:25:28,406 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:25:28,407 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:25:28,407 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:25:29,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:25:30,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:32,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:34,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:36,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:38,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:39,109 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:25:42,236 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:25:42,237 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:25:48,476 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:25:48,477 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:25:48,477 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:25:49,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:25:50,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:52,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:54,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:56,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:25:57,368 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:25:57,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:25:58,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:08,561 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:26:08,562 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:26:08,562 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:26:09,174 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:26:09,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:26:10,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:12,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:12,499 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:26:12,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:26:14,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:16,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:27,671 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:26:27,671 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:26:28,601 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:26:28,601 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:26:28,602 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:26:29,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:26:30,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:32,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:34,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:36,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:39,242 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:26:42,800 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:26:42,800 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:26:48,654 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:26:48,655 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:26:48,656 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:26:49,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:26:50,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:52,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:54,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:56,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:26:57,936 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:26:57,936 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:26:58,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:08,703 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:27:08,704 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:27:08,704 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:27:09,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:27:09,312 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:27:10,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:12,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:13,094 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:27:13,095 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:27:14,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:16,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:18,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:28,310 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:27:28,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:27:28,794 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:27:28,795 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:27:28,795 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:27:29,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:27:30,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:32,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:34,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:36,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:39,381 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:27:43,444 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:27:43,445 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:27:48,829 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:27:48,829 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:27:48,830 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:27:49,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:27:50,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:52,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:54,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:56,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:27:58,575 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:27:58,575 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:28:08,876 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:28:08,876 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:28:08,877 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:28:09,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:28:09,454 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:28:10,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:12,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:13,704 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:28:13,705 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:28:14,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:16,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:18,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:28,835 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:28:28,836 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:28:28,964 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:28:28,965 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:28:28,965 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:28:29,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:28:30,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:32,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:34,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:36,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:38,280 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:39,525 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:28:43,977 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:28:43,978 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:28:48,979 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:28:48,980 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:28:48,980 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:28:49,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:28:50,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:53,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:55,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:57,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:28:59,110 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:28:59,110 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:29:09,019 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:29:09,019 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:29:09,020 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:29:09,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:29:09,594 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:29:10,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:11,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:13,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:14,243 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:29:14,243 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:29:15,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:17,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:29,073 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:29:29,073 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:29:29,074 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:29:29,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:29:29,413 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:29:29,414 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:29:30,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:31,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:33,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:35,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:37,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:39,667 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:29:44,571 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:29:44,571 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:29:49,118 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:29:49,118 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:29:49,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:29:49,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:29:50,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:51,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:53,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:55,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:57,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:29:59,701 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:29:59,702 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:30:09,172 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:30:09,173 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:30:09,173 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:30:09,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:30:09,743 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:30:10,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:11,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:13,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:14,831 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:30:14,831 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:30:15,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:17,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:29,231 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:30:29,232 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:30:29,232 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:30:29,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:30:29,997 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:30:29,997 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:30:31,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:33,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:35,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:37,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:39,816 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:30:45,134 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:30:45,135 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:30:49,303 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:30:49,304 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:30:49,304 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:30:49,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:30:51,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:53,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:55,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:30:57,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:00,265 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:31:00,266 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:31:09,417 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:31:09,418 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:31:09,419 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:31:09,893 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:31:10,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:31:11,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:13,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:15,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:15,403 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:31:15,404 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:31:17,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:29,541 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:31:29,541 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:31:29,542 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:31:30,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:31:30,559 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:31:30,560 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:31:31,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:33,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:35,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:37,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:39,968 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:31:45,729 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:31:45,730 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:31:49,585 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:31:49,585 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:31:49,586 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:31:50,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:31:51,366 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:53,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:55,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:31:57,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:00,862 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:32:00,862 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:32:09,645 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:32:09,645 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:32:09,646 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:32:10,043 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:32:10,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:32:11,375 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:13,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:15,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:15,994 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:32:15,995 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:32:17,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:29,688 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:32:29,688 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:32:29,689 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:32:30,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:32:31,138 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:32:31,139 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:32:31,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:33,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:35,385 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:37,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:40,118 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:32:46,297 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:32:46,297 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:32:49,771 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:32:49,771 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:32:49,772 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:32:50,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:32:51,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:53,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:55,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:32:57,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:01,428 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:33:01,428 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:33:09,844 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:33:09,844 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:33:09,845 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:33:10,191 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:33:10,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:33:11,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:14,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:16,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:16,559 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:33:16,560 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:33:18,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:29,935 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:33:29,935 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:33:29,935 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:33:30,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:33:31,742 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:33:31,742 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:33:32,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:34,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:36,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:38,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:40,269 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:33:46,879 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:33:46,879 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:33:49,973 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:33:49,974 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:33:49,975 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:33:50,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:33:52,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:54,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:56,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:33:58,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:02,016 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:34:02,017 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:34:10,027 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:34:10,027 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:34:10,028 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:34:10,344 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:34:10,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:34:12,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:14,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:16,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:17,149 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:34:17,150 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:34:18,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:30,071 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:34:30,071 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:34:30,072 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:34:30,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:34:32,287 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:34:32,288 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:34:32,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:34,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:36,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:38,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:40,419 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:34:47,459 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:34:47,460 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:34:50,160 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:34:50,161 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:34:50,161 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:34:50,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:34:52,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:54,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:56,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:34:58,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:02,591 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:35:02,591 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:35:10,250 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:35:10,251 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:35:10,251 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:35:10,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:35:10,494 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:35:12,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:14,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:16,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:17,721 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:35:17,722 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:35:18,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:30,304 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:35:30,304 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:35:30,305 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:35:30,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:35:32,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:32,908 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:35:32,908 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:35:34,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:36,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:38,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:40,566 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:35:48,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:35:48,077 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:35:50,351 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:35:50,351 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:35:50,352 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:35:50,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:35:52,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:54,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:56,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:35:58,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:03,208 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:36:03,208 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:36:10,443 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:36:10,444 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:36:10,444 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:36:10,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:36:10,642 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:36:12,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:14,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:16,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:18,342 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:36:18,342 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:36:18,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:30,499 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:36:30,500 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:36:30,500 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:36:31,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:36:32,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:33,511 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:36:33,511 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:36:34,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:36,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:38,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:40,716 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:36:48,646 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:36:48,646 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:36:50,557 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:36:50,557 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:36:50,557 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:36:51,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:36:52,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:54,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:56,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:36:58,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:03,785 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:37:03,785 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:37:10,618 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:37:10,619 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:37:10,619 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:37:10,792 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:37:11,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:37:12,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:14,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:16,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:18,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:18,918 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:37:18,919 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:37:30,780 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:37:30,781 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:37:30,781 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:37:31,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:37:32,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:34,048 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:37:34,049 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:37:34,521 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:36,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:38,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:40,862 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:37:49,185 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:37:49,186 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:37:50,865 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:37:50,866 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:37:50,866 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:37:51,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:37:52,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:54,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:56,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:37:58,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:04,316 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:38:04,317 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:38:10,904 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:38:10,905 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:38:10,907 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:38:10,938 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:38:11,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:38:12,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:14,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:16,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:18,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:19,447 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:38:19,448 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:38:30,953 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:38:30,954 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:38:30,954 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:38:31,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:38:32,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:33,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:34,700 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:38:34,701 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:38:35,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:37,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:39,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:41,010 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:38:49,832 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:38:49,833 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:38:51,031 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:38:51,031 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:38:51,032 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:38:51,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:38:53,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:55,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:57,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:38:59,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:04,962 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:39:04,962 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:39:11,070 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:39:11,071 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:39:11,073 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:39:11,080 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:39:11,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:39:13,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:15,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:17,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:19,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:20,093 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:39:20,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:39:31,114 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:39:31,115 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:39:31,115 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:39:31,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:39:33,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:35,274 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:39:35,274 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:39:35,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:37,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:39,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:41,151 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:39:50,404 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:39:50,404 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:39:51,178 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:39:51,179 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:39:51,179 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:39:51,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:39:53,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:55,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:57,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:39:59,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:05,536 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:40:05,536 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:40:11,221 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:40:11,262 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:40:11,263 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:40:11,263 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:40:11,586 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:40:13,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:15,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:17,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:19,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:20,670 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:40:20,670 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:40:31,353 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:40:31,354 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:40:31,354 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:40:31,595 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:40:33,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:35,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:35,802 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:40:35,803 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:40:37,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:39,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:41,293 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:40:50,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:40:50,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:40:51,428 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:40:51,428 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:40:51,429 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:40:51,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:40:53,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:55,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:57,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:40:59,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:06,075 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:41:06,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:41:11,366 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:41:11,522 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:41:11,523 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:41:11,523 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:41:11,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:41:13,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:15,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:17,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:19,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:21,205 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:41:21,205 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:41:31,589 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:41:31,590 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:41:31,590 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:41:31,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:41:33,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:35,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:36,338 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:41:36,339 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:41:37,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:39,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:41,437 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:41:51,471 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:41:51,472 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:41:51,661 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:41:51,662 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:41:51,662 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:41:52,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:41:53,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:55,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:57,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:41:59,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:06,603 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:42:06,603 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:42:11,510 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:42:11,773 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:42:11,773 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:42:11,774 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:42:12,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:42:13,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:15,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:17,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:19,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:21,735 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:42:21,735 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:42:31,863 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:42:31,864 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:42:31,864 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:42:32,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:42:33,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:35,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:36,869 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:42:36,869 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:42:37,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:39,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:41,580 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:42:51,919 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:42:51,919 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:42:51,920 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:42:52,013 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:42:52,013 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:42:52,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:42:53,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:55,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:57,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:42:59,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:07,178 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:43:07,179 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:43:11,654 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:43:12,010 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:43:12,010 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:43:12,011 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:43:12,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:43:13,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:14,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:16,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:18,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:20,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:22,310 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:43:22,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:43:32,074 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:43:32,074 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:43:32,075 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:43:32,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:43:33,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:34,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:36,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:37,438 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:43:37,439 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:43:38,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:40,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:41,725 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:43:52,114 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:43:52,115 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:43:52,115 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:43:52,609 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:43:52,609 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:43:52,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:43:53,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:54,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:56,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:43:58,686 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:00,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:07,761 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:44:07,761 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:44:11,794 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:44:12,183 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:44:12,183 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:44:12,183 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:44:12,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:44:13,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:14,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:16,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:18,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:20,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:22,895 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:44:22,895 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:44:32,226 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:44:32,226 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:44:32,227 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:44:32,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:44:34,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:36,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:38,026 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:44:38,026 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:44:38,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:40,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:41,869 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:44:52,294 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:44:52,294 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:44:52,295 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:44:52,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:44:53,159 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:44:53,159 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:44:54,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:56,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:44:58,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:00,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:08,325 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:45:08,330 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:45:11,944 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:45:12,358 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:45:12,358 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:45:12,359 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:45:12,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:45:14,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:16,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:18,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:20,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:23,465 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:45:23,466 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:45:32,508 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:45:32,509 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:45:32,509 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:45:32,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:45:34,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:36,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:38,607 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:45:38,607 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:45:38,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:40,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:42,020 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:45:52,556 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:45:52,557 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:45:52,557 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:45:52,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:45:53,816 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:45:53,816 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:45:54,740 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:56,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:45:58,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:00,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:08,983 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:46:08,984 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:46:12,095 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:46:12,622 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:46:12,622 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:46:12,623 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:46:12,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:46:14,749 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:16,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:18,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:20,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:24,115 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:46:24,116 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:46:32,699 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:46:32,699 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:46:32,700 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:46:32,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:46:34,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:36,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:38,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:39,254 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:46:39,255 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:46:40,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:42,170 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:46:52,842 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:46:52,842 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:46:52,843 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:46:53,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:46:54,397 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:46:54,398 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:46:54,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:56,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:46:58,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:00,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:09,554 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:47:09,554 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:47:12,246 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:47:12,892 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:47:12,893 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:47:12,893 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:47:13,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:47:14,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:16,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:18,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:20,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:24,684 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:47:24,684 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:47:32,984 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:47:32,984 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:47:32,985 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:47:33,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:47:34,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:36,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:38,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:39,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:47:39,814 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:47:40,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:42,317 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:47:53,079 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:47:53,079 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:47:53,080 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:47:53,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:47:54,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:55,047 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:47:55,047 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:47:55,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:57,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:47:59,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:01,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:10,208 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:48:10,208 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:48:12,391 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:48:13,121 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:48:13,121 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:48:13,121 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:48:13,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:48:14,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:15,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:17,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:19,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:21,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:25,340 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:48:25,341 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:48:33,199 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:48:33,200 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:48:33,200 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:48:33,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:48:34,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:35,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:37,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:39,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:40,476 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:48:40,476 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:48:41,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:42,463 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:48:53,250 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:48:53,250 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:48:53,251 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:48:53,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:48:54,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:55,671 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:48:55,671 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:48:55,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:57,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:48:59,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:01,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:10,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:49:10,813 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:49:12,537 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:49:13,312 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:49:13,312 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:49:13,313 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:49:13,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:49:14,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:15,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:17,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:19,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:21,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:25,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:49:25,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:49:33,352 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:49:33,352 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:49:33,353 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:49:33,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:49:34,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:35,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:37,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:39,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:41,076 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:49:41,076 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:49:41,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:42,611 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:49:53,421 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:49:53,421 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:49:53,421 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:49:53,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:49:54,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:55,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:56,209 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:49:56,210 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:49:57,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:49:59,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:01,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:11,355 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:50:11,355 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:50:12,682 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:50:13,459 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:50:13,460 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:50:13,461 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:50:13,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:50:15,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:17,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:19,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:21,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:26,486 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:50:26,486 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:50:33,528 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:50:33,529 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:50:33,529 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:50:33,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:50:35,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:37,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:39,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:41,617 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:50:41,618 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:50:41,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:42,756 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:50:53,573 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:50:53,573 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:50:53,574 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:50:53,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:50:55,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:56,748 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:50:56,748 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:50:57,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:50:59,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:01,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:11,932 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:51:11,932 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:51:12,827 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:51:13,625 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:51:13,625 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:51:13,626 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:51:13,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:51:15,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:17,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:19,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:21,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:27,071 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:51:27,071 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:51:33,815 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:51:33,815 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:51:33,816 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:51:33,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:51:35,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:37,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:39,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:41,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:42,208 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:51:42,209 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:51:42,902 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:51:53,887 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:51:53,887 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:51:53,888 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:51:53,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:51:55,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:57,358 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:51:57,359 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:51:57,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:51:59,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:01,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:12,496 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:52:12,497 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:52:12,976 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:52:13,929 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:52:13,929 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:52:13,929 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:52:14,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:52:15,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:17,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:19,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:21,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:27,632 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:52:27,633 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:52:34,020 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:52:34,020 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:52:34,021 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:52:34,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:52:35,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:38,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:40,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:42,762 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:52:42,763 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:52:42,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:43,047 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:52:54,083 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:52:54,083 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:52:54,084 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:52:54,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:52:55,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:56,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:52:57,902 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:52:57,902 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:52:58,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:00,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:02,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:13,033 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:53:13,033 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:53:13,164 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:53:14,180 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:53:14,180 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:53:14,181 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:53:14,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:53:15,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:16,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:18,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:20,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:22,942 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:28,166 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:53:28,166 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:53:34,240 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:53:34,240 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:53:34,241 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:53:34,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:53:35,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:36,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:38,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:40,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:42,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:43,193 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:53:43,300 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:53:43,300 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:53:54,316 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:53:54,316 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:53:54,317 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:53:54,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:53:55,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:56,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:53:58,435 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:53:58,435 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:53:58,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:00,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:02,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:13,269 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:54:13,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:54:13,566 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:54:14,413 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:54:14,413 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:54:14,414 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:54:14,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:54:15,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:16,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:18,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:20,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:22,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:28,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:54:28,696 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:54:34,482 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:54:34,482 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:54:34,483 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:54:34,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:54:35,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:36,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:38,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:40,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:42,980 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:43,341 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:54:43,834 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:54:43,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:54:54,540 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:54:54,540 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:54:54,540 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:54:54,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:54:55,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:56,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:54:58,964 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:54:58,964 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:54:58,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:00,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:02,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:13,412 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:55:14,101 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:55:14,101 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:55:14,607 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:55:14,607 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:55:14,608 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:55:14,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:55:15,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:16,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:18,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:20,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:22,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:29,235 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:55:29,235 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:55:34,660 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:55:34,660 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:55:34,661 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:55:35,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:55:37,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:39,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:41,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:43,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:43,482 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:55:44,371 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:55:44,372 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:55:54,697 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:55:54,698 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:55:54,698 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:55:55,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:55:57,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:59,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:55:59,533 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:55:59,534 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:56:01,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:03,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:05,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:07,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:09,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:11,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:13,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:13,546 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:56:14,663 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:56:14,664 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:56:15,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:17,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:19,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:22,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:24,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:26,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:28,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:29,795 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:56:29,795 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:56:30,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:32,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:34,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:36,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:38,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:40,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:42,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:43,613 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:56:44,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:44,952 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:56:44,953 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:56:46,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:48,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:50,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:52,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:54,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:56,043 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:56:58,633 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:56:58,634 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:56:58,634 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:56:59,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:57:00,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:00,093 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:57:00,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:57:02,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:04,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:06,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:08,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:13,678 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:57:15,230 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:57:15,230 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:57:18,706 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:57:18,706 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:57:18,707 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:57:19,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:57:20,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:21,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:22,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:24,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:26,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:28,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:30,368 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:57:30,369 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:57:38,773 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:57:38,773 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:57:38,774 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:57:39,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:57:40,063 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:41,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:42,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:43,747 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:57:44,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:45,499 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:57:45,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:57:46,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:48,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:57:58,824 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:57:58,825 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:57:58,825 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:57:59,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:58:00,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:00,634 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:58:00,635 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:58:01,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:02,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:04,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:06,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:08,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:13,813 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:58:15,773 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:58:15,774 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:58:18,883 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:58:18,883 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:58:18,883 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:58:19,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:58:20,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:21,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:22,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:24,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:26,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:28,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:30,903 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:58:30,904 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:58:38,924 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:58:38,925 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:58:38,925 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:58:39,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:39,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:58:40,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:42,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:43,880 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:58:44,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:46,035 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:58:46,036 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:58:46,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:48,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:58:59,012 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:58:59,013 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:58:59,013 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:58:59,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:59:00,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:01,193 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:59:01,194 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:59:02,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:04,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:06,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:08,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:13,948 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:59:16,376 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:59:16,377 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:59:19,054 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:59:19,054 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:59:19,055 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:59:19,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:59:20,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:22,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:24,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:26,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:31,509 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:59:31,510 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:59:39,117 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:59:39,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:59:39,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 07:59:40,115 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:40,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 07:59:42,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:43,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:44,017 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 07:59:45,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:47,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 07:59:47,495 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 07:59:47,496 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 07:59:59,187 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 07:59:59,187 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 07:59:59,187 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:00:00,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:00:01,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:02,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:02,671 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:00:02,671 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:00:03,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:05,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:07,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:14,089 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:00:17,815 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:00:17,816 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:00:19,244 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:00:19,245 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:00:19,245 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:00:20,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:20,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:00:21,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:23,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:25,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:27,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:32,946 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:00:32,947 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:00:39,289 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:00:39,289 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:00:39,290 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:00:40,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:00:41,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:42,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:43,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:44,160 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:00:45,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:47,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:00:48,077 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:00:48,078 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:00:59,333 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:00:59,333 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:00:59,334 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:01:00,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:01:01,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:02,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:03,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:03,211 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:01:03,211 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:01:05,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:07,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:09,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:14,232 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:01:18,342 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:01:18,343 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:01:19,405 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:01:19,406 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:01:19,406 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:01:20,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:01:21,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:22,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:23,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:25,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:27,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:33,474 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:01:33,475 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:01:39,480 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:01:39,481 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:01:39,481 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:01:40,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:40,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:01:41,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:43,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:44,306 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:01:45,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:47,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:48,605 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:01:48,605 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:01:49,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:01:59,598 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:01:59,598 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:01:59,599 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:02:00,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:00,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:02:01,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:03,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:03,762 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:02:03,763 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:02:05,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:07,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:14,382 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:02:18,891 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:02:18,891 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:02:19,670 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:02:19,670 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:02:19,671 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:02:20,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:02:21,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:22,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:23,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:25,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:27,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:34,021 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:02:34,022 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:02:39,757 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:02:39,757 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:02:39,758 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:02:40,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:02:41,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:42,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:43,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:44,456 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:02:45,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:47,201 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:02:49,153 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:02:49,153 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:02:59,806 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:02:59,807 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:02:59,808 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:03:00,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:03:01,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:02,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:03,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:04,284 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:03:04,284 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:03:05,208 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:07,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:09,210 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:14,533 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:03:19,414 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:03:19,414 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:03:19,903 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:03:19,904 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:03:19,904 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:03:20,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:03:21,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:22,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:23,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:26,217 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:28,218 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:30,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:34,546 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:03:34,547 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:03:39,955 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:03:39,956 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:03:39,956 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:03:40,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:03:42,224 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:44,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:44,606 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:03:46,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:48,227 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:03:49,679 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:03:49,679 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:04:00,051 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:04:00,052 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:04:00,052 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:04:00,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:04:02,233 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:04,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:04,823 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:04:04,824 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:04:06,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:08,236 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:14,685 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:04:19,978 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:04:19,979 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:04:20,131 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:04:20,131 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:04:20,132 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:04:20,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:04:22,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:23,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:24,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:26,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:28,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:35,109 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:04:35,109 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:04:40,232 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:04:40,232 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:04:40,233 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:04:40,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:04:42,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:43,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:44,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:44,757 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:04:46,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:48,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:04:50,244 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:04:50,244 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:05:00,308 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:05:00,308 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:05:00,309 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:05:01,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:05:02,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:03,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:04,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:05,376 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:05:05,377 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:05:06,261 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:08,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:14,835 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:05:20,389 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:05:20,389 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:05:20,390 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:05:20,523 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:05:20,523 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:05:21,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:05:22,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:23,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:24,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:26,270 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:28,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:35,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:05:35,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:05:40,453 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:05:40,453 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:05:40,454 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:05:41,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:05:42,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:43,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:44,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:44,907 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:05:46,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:48,279 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:05:50,795 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:05:50,795 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:06:00,493 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:06:00,493 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:06:00,494 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:06:01,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:06:02,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:03,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:04,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:05,928 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:06:05,928 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:06:06,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:08,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:14,982 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:06:20,533 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:06:20,534 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:06:20,534 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:06:21,120 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:06:21,120 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:06:21,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:06:22,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:23,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:24,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:26,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:28,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:36,253 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:06:36,254 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:06:40,588 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:06:40,588 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:06:40,588 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:06:41,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:41,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:06:42,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:44,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:45,059 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:06:46,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:48,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:06:51,391 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:06:51,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:07:00,682 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:07:00,683 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:07:00,683 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:07:01,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:01,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:07:02,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:04,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:06,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:06,523 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:07:06,524 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:07:08,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:15,131 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:07:20,745 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:07:20,745 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:07:20,746 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:07:21,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:21,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:07:21,675 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:07:21,675 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:07:22,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:24,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:26,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:28,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:36,821 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:07:36,822 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:07:40,837 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:07:40,837 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:07:40,838 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:07:41,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:07:42,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:43,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:45,203 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:07:45,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:47,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:49,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:07:51,951 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:07:51,951 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:08:00,876 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:08:00,876 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:08:00,877 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:08:01,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:08:03,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:05,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:07,084 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:08:07,084 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:08:07,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:09,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:15,277 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:08:20,970 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:08:20,970 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:08:20,971 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:08:21,347 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:08:22,215 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:08:22,215 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:08:23,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:25,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:27,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:29,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:37,355 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:08:37,355 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:08:41,081 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:08:41,081 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:08:41,082 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:08:41,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:41,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:08:43,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:45,350 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:08:45,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:47,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:49,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:08:52,489 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:08:52,490 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:09:01,137 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:09:01,137 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:09:01,137 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:09:01,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:09:03,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:05,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:07,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:07,624 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:09:07,624 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:09:09,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:15,426 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:09:21,231 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:09:21,231 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:09:21,232 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:09:21,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:09:22,803 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:09:22,804 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:09:23,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:25,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:27,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:29,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:37,953 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:09:37,954 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:09:41,276 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:09:41,276 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:09:41,276 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:09:41,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:09:43,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:44,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:45,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:45,497 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:09:47,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:49,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:09:53,094 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:09:53,095 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:10:01,364 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:10:01,365 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:10:01,365 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:10:01,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:10:03,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:04,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:05,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:07,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:08,266 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:10:08,267 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:10:09,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:15,573 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:10:21,422 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:10:21,423 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:10:21,423 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:10:22,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:10:23,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:23,406 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:10:23,406 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:10:24,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:25,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:27,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:29,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:38,562 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:10:38,562 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:10:41,602 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:10:41,602 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:10:41,603 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:10:42,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:10:43,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:44,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:45,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:45,639 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:10:47,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:49,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:10:53,692 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:10:53,692 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:11:01,686 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:11:01,686 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:11:01,687 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:11:02,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:11:03,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:04,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:05,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:07,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:08,822 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:11:08,822 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:11:09,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:15,707 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:11:21,776 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:11:21,776 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:11:21,777 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:11:22,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:11:23,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:23,960 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:11:23,960 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:11:24,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:25,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:27,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:29,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:39,100 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:11:39,101 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:11:41,838 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:11:41,839 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:11:41,839 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:11:42,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:11:43,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:44,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:45,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:45,777 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:11:47,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:49,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:11:54,232 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:11:54,233 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:12:01,883 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:12:01,884 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:12:01,884 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:12:02,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:12:03,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:04,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:05,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:08,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:09,364 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:12:09,364 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:12:10,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:15,843 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:12:21,949 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:12:21,949 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:12:21,950 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:12:22,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:12:24,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:24,499 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:12:24,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:12:26,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:28,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:30,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:39,649 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:12:39,649 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:12:42,022 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:12:42,023 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:12:42,023 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:12:42,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:12:44,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:45,911 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:12:46,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:48,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:50,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:12:54,781 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:12:54,781 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:13:02,061 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:13:02,061 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:13:02,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:13:02,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:13:04,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:06,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:08,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:09,912 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:13:09,913 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:13:10,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:15,981 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:13:22,124 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:13:22,125 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:13:22,125 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:13:22,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:13:24,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:25,058 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:13:25,059 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:13:26,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:28,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:30,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:40,199 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:13:40,200 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:13:42,208 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:13:42,209 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:13:42,209 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:13:42,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:13:44,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:46,051 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:13:46,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:48,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:50,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:13:55,333 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:13:55,333 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:14:02,261 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:14:02,261 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:14:02,262 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:14:02,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:14:04,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:06,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:08,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:10,462 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:14:10,463 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:14:10,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:16,120 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:14:22,311 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:14:22,311 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:14:22,312 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:14:22,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:14:24,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:25,676 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:14:25,676 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:14:26,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:28,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:30,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:40,875 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:14:40,875 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:14:42,411 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:14:42,412 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:14:42,412 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:14:42,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:14:44,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:46,188 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:14:46,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:48,521 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:50,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:14:56,006 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:14:56,006 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:15:02,481 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:15:02,482 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:15:02,482 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:15:02,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:15:04,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:06,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:08,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:10,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:11,137 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:15:11,137 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:15:16,261 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:15:22,533 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:15:22,534 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:15:22,535 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:15:22,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:15:24,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:25,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:26,296 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:15:26,297 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:15:26,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:28,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:30,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:41,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:15:41,433 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:15:42,597 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:15:42,597 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:15:42,598 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:15:43,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:15:44,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:45,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:46,333 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:15:46,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:48,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:50,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:15:56,563 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:15:56,563 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:16:02,721 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:16:02,721 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:16:02,722 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:16:03,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:16:04,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:05,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:06,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:08,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:10,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:11,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:16:11,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:16:16,407 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:16:22,761 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:16:22,761 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:16:22,762 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:16:23,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:16:24,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:25,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:26,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:27,684 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:16:27,684 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:16:28,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:30,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:42,815 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:16:42,815 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:16:42,944 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:16:42,945 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:16:42,945 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:16:43,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:16:44,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:45,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:46,475 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:16:46,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:48,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:50,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:16:57,991 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:16:57,991 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:17:02,878 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:17:02,879 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:17:02,879 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:17:03,576 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:17:05,577 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:07,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:09,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:11,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:13,128 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:17:13,128 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:17:16,542 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:17:22,946 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:17:22,947 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:17:22,947 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:17:23,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:17:25,586 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:27,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:28,288 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:17:28,289 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:17:29,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:31,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:43,035 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:17:43,035 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:17:43,036 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:17:43,467 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:17:43,468 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:17:43,595 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:17:45,595 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:46,617 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:17:47,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:49,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:51,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:17:58,631 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:17:58,631 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:18:03,081 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:18:03,082 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:18:03,082 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:18:03,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:18:05,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:07,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:09,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:11,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:13,823 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:18:13,824 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:18:16,690 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:18:23,148 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:18:23,149 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:18:23,149 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:18:23,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:23,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:18:25,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:27,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:28,957 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:18:28,958 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:18:29,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:31,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:43,195 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:18:43,196 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:18:43,197 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:18:43,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:18:44,127 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:18:44,128 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:18:45,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:46,764 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:18:47,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:49,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:51,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:18:59,258 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:18:59,258 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:19:03,253 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:19:03,254 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:19:03,254 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:19:03,629 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:19:05,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:07,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:09,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:11,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:14,391 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:19:14,392 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:19:16,837 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:19:23,293 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:19:23,293 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:19:23,294 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:19:23,638 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:19:25,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:27,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:29,525 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:19:29,525 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:19:29,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:31,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:43,414 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:19:43,414 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:19:43,414 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:19:43,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:19:44,658 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:19:44,659 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:19:45,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:46,910 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:19:47,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:49,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:51,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:19:59,818 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:19:59,819 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:20:03,464 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:20:03,465 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:20:03,465 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:20:03,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:20:05,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:07,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:09,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:11,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:14,950 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:20:14,950 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:20:16,984 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:20:23,559 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:20:23,559 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:20:23,560 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:20:23,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:20:25,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:26,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:27,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:29,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:30,090 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:20:30,091 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:20:31,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:43,627 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:20:43,627 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:20:43,627 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:20:43,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:20:45,230 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:20:45,230 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:20:45,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:46,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:47,054 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:20:47,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:49,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:20:51,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:00,369 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:21:00,369 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:21:03,679 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:21:03,680 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:21:03,680 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:21:03,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:21:05,682 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:06,682 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:07,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:09,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:11,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:15,501 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:21:15,501 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:21:17,132 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:21:23,778 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:21:23,779 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:21:23,779 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:21:24,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:21:25,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:26,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:27,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:29,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:30,631 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:21:30,631 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:21:31,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:43,836 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:21:43,837 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:21:43,837 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:21:44,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:21:45,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:45,822 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:21:45,822 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:21:46,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:47,204 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:21:47,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:49,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:21:51,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:00,971 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:22:00,971 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:22:03,890 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:22:03,891 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:22:03,891 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:22:04,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:22:05,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:06,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:07,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:09,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:11,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:16,104 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:22:16,105 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:22:17,280 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:22:23,948 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:22:23,948 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:22:23,948 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:22:24,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:24,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:22:25,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:27,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:29,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:31,237 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:22:31,237 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:22:32,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:44,003 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:22:44,004 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:22:44,004 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:22:44,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:44,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:22:46,367 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:22:46,367 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:22:46,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:47,352 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:22:48,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:50,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:22:52,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:01,539 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:23:01,539 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:23:04,054 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:23:04,054 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:23:04,055 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:23:04,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:23:06,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:08,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:10,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:12,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:16,672 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:23:16,673 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:23:17,424 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:23:24,145 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:23:24,145 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:23:24,146 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:23:24,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:23:26,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:28,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:30,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:32,002 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:23:32,002 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:23:32,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:44,234 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:23:44,234 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:23:44,235 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:23:44,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:23:46,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:47,166 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:23:47,166 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:23:47,502 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:23:48,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:50,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:23:52,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:02,310 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:24:02,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:24:04,291 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:24:04,291 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:24:04,292 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:24:04,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:24:06,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:08,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:10,764 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:12,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:17,452 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:24:17,453 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:24:17,588 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:24:24,334 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:24:24,335 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:24:24,335 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:24:24,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:24:26,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:28,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:30,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:32,589 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:24:32,590 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:24:32,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:44,409 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:24:44,409 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:24:44,410 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:24:44,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:24:46,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:47,652 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:24:47,769 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:24:47,769 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:24:48,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:50,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:24:52,782 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:03,091 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:25:03,092 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:25:04,462 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:25:04,462 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:25:04,462 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:25:04,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:25:06,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:08,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:10,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:12,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:17,721 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:25:18,222 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:25:18,222 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:25:24,499 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:25:24,500 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:25:24,500 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:25:24,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:25:26,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:28,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:30,799 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:32,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:33,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:25:33,354 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:25:44,562 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:25:44,562 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:25:44,563 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:25:44,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:25:46,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:47,792 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:25:47,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:48,486 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:25:48,486 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:25:48,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:50,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:25:52,809 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:03,616 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:26:03,616 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:26:04,648 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:26:04,648 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:26:04,649 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:26:04,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:26:06,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:07,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:08,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:10,818 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:12,818 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:17,866 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:26:18,753 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:26:18,753 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:26:24,694 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:26:24,694 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:26:24,695 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:26:24,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:26:26,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:27,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:28,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:30,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:32,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:33,883 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:26:33,883 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:26:44,780 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:26:44,781 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:26:44,781 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:26:44,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:26:45,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:46,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:47,932 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:26:48,835 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:49,012 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:26:49,012 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:26:50,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:26:52,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:04,143 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:27:04,144 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:27:04,862 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:27:04,863 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:27:04,863 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:27:05,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:27:06,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:07,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:08,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:10,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:12,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:17,999 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:27:19,274 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:27:19,275 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:27:24,940 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:27:24,941 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:27:24,941 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:27:25,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:27:26,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:27,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:28,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:30,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:32,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:34,403 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:27:34,403 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:27:44,980 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:27:44,981 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:27:44,981 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:27:45,861 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:27:46,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:47,862 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:48,066 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:27:48,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:49,546 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:27:49,546 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:27:50,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:27:52,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:04,683 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:28:04,683 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:28:05,071 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:28:05,071 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:28:05,072 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:28:05,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:05,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:28:06,870 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:08,871 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:10,872 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:12,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:18,132 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:28:19,815 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:28:19,816 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:28:25,166 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:28:25,166 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:28:25,167 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:28:25,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:28:27,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:29,880 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:31,881 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:33,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:34,950 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:28:34,950 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:28:45,251 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:28:45,252 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:28:45,252 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:28:45,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:28:47,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:48,204 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:28:49,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:50,083 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:28:50,084 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:28:51,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:28:53,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:05,213 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:29:05,213 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:29:05,344 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:29:05,345 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:29:05,345 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:29:05,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:29:07,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:09,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:11,900 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:13,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:18,272 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:29:20,365 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:29:20,366 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:29:25,366 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:29:25,366 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:29:25,367 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:29:25,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:29:27,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:29,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:31,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:33,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:35,506 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:29:35,506 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:29:45,412 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:29:45,412 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:29:45,412 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:29:45,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:29:47,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:48,343 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:29:49,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:50,647 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:29:50,647 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:29:51,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:29:53,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:05,475 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:30:05,475 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:30:05,476 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:30:05,784 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:30:05,785 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:30:05,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:30:07,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:09,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:11,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:13,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:18,417 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:30:20,920 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:30:20,921 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:30:25,559 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:30:25,559 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:30:25,560 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:30:25,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:25,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:30:27,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:29,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:31,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:33,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:36,052 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:30:36,053 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:30:45,614 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:30:45,614 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:30:45,615 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:30:45,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:30:46,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:47,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:48,493 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:30:49,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:51,189 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:30:51,189 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:30:51,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:30:53,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:05,670 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:31:05,670 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:31:05,671 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:31:05,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:31:06,350 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:31:06,350 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:31:06,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:07,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:09,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:11,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:13,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:18,566 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:31:21,500 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:31:21,500 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:31:25,725 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:31:25,726 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:31:25,726 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:31:25,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:31:26,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:27,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:29,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:31,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:33,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:36,633 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:31:36,633 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:31:45,766 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:31:45,767 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:31:45,767 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:31:45,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:31:47,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:48,638 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:31:48,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:49,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:51,764 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:31:51,765 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:31:51,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:31:53,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:05,832 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:32:05,833 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:32:05,833 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:32:05,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:32:06,971 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:32:06,972 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:32:07,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:08,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:09,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:11,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:13,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:18,712 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:32:22,151 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:32:22,151 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:32:25,899 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:32:25,900 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:32:25,900 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:32:25,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:32:27,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:28,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:29,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:31,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:33,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:37,281 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:32:37,286 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:32:45,953 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:32:45,954 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:32:45,954 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:32:46,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:32:47,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:48,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:48,782 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:32:50,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:52,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:32:52,463 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:32:52,464 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:32:54,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:06,006 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:33:06,007 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:33:06,007 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:33:06,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:33:07,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:07,618 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:33:07,619 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:33:08,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:10,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:12,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:14,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:18,858 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:33:22,775 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:33:22,775 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:33:26,061 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:33:26,062 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:33:26,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:33:27,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:27,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:33:28,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:30,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:32,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:34,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:37,910 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:33:37,910 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:33:46,105 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:33:46,105 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:33:46,106 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:33:47,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:47,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:33:48,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:48,929 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:33:50,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:52,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:33:53,038 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:33:53,039 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:33:54,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:06,170 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:34:06,170 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:34:06,170 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:34:07,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:07,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:34:08,226 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:34:08,226 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:34:09,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:11,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:13,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:15,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:19,003 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:34:23,430 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:34:23,430 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:34:26,235 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:34:26,235 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:34:26,235 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:34:27,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:27,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:34:29,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:31,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:33,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:35,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:38,558 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:34:38,559 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:34:46,319 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:34:46,320 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:34:46,320 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:34:47,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:47,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:34:49,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:49,080 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:34:51,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:53,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:34:53,692 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:34:53,693 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:34:55,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:06,382 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:35:06,383 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:35:06,383 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:35:07,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:07,064 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:35:08,868 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:35:08,869 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:35:09,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:11,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:13,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:15,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:19,152 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:35:24,058 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:35:24,058 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:35:26,449 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:35:26,449 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:35:26,450 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:35:27,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:27,073 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:35:29,074 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:31,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:33,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:35,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:39,188 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:35:39,189 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:35:46,488 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:35:46,489 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:35:46,489 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:35:47,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:47,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:35:49,083 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:49,225 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:35:51,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:53,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:35:54,320 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:35:54,320 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:35:55,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:06,564 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:36:06,564 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:36:06,564 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:36:07,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:36:08,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:09,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:09,449 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:36:09,450 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:36:11,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:13,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:15,095 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:19,297 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:36:24,581 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:36:24,582 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:36:26,616 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:36:26,617 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:36:26,617 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:36:27,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:36:28,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:29,101 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:31,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:33,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:35,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:39,713 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:36:39,713 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:36:46,655 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:36:46,656 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:36:46,656 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:36:47,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:36:48,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:49,110 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:49,369 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:36:51,111 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:53,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:36:54,845 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:36:54,846 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:36:55,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:06,719 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:37:06,719 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:37:06,719 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:37:07,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:37:08,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:09,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:09,977 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:37:09,977 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:37:11,119 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:13,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:15,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:19,444 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:37:25,130 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:37:25,130 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:37:26,789 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:37:26,789 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:37:26,790 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:37:27,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:37:28,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:29,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:31,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:33,128 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:35,129 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:40,263 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:37:40,263 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:37:46,838 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:37:46,839 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:37:46,839 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:37:47,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:37:48,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:49,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:49,517 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:37:51,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:53,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:55,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:37:55,393 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:37:55,393 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:38:06,877 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:38:06,878 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:38:06,878 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:38:07,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:38:08,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:09,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:10,539 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:38:10,539 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:38:11,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:13,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:15,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:19,595 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:38:25,678 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:38:25,678 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:38:26,914 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:38:26,914 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:38:26,915 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:38:27,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:38:28,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:29,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:31,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:33,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:35,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:40,809 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:38:40,810 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:38:46,981 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:38:46,981 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:38:46,982 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:38:47,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:38:48,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:49,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:49,667 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:38:51,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:53,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:55,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:38:55,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:38:55,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:39:07,058 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:39:07,059 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:39:07,059 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:39:07,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:39:08,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:09,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:11,075 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:39:11,075 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:39:11,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:13,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:15,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:19,740 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:39:26,207 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:39:26,207 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:39:27,101 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:39:27,101 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:39:27,102 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:39:27,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:39:28,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:29,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:31,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:33,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:35,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:41,339 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:39:41,339 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:39:47,141 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:39:47,141 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:39:47,141 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:39:47,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:39:48,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:49,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:49,814 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:39:51,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:53,189 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:55,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:39:56,474 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:39:56,474 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:40:07,205 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:40:07,205 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:40:07,205 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:40:08,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:08,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:40:09,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:11,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:11,604 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:40:11,605 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:40:13,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:15,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:19,886 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:40:26,736 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:40:26,736 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:40:27,263 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:40:27,263 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:40:27,264 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:40:28,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:28,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:40:29,204 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:31,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:33,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:35,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:41,870 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:40:41,871 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:40:47,315 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:40:47,315 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:40:47,316 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:40:48,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:40:49,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:49,960 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:40:51,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:53,214 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:55,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:40:57,004 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:40:57,004 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:41:07,394 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:41:07,394 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:41:07,395 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:41:08,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:41:09,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:11,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:12,132 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:41:12,133 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:41:13,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:15,223 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:20,029 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:41:27,292 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:41:27,292 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:41:27,438 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:41:27,438 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:41:27,439 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:41:28,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:41:29,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:30,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:32,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:34,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:36,232 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:42,429 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:41:42,429 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:41:47,565 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:41:47,565 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:41:47,566 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:41:48,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:41:49,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:50,100 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:41:50,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:52,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:54,241 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:56,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:41:57,564 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:41:57,564 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:42:07,619 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:42:07,619 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:42:07,620 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:42:08,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:42:09,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:10,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:12,249 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:12,695 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:42:12,695 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:42:14,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:16,251 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:20,169 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:42:27,670 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:42:27,670 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:42:27,671 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:42:27,835 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:42:27,836 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:42:28,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:42:29,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:30,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:32,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:34,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:36,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:42,988 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:42:42,988 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:42:47,798 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:42:47,798 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:42:47,799 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:42:48,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:42:49,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:50,234 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:42:50,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:52,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:54,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:56,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:42:58,121 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:42:58,121 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:43:07,846 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:43:07,846 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:43:07,847 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:43:08,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:43:09,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:10,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:12,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:13,253 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:43:13,254 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:43:14,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:16,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:20,301 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:43:27,887 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:43:27,887 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:43:27,888 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:43:28,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:43:28,389 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:43:28,390 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:43:29,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:30,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:32,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:34,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:36,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:43,520 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:43:43,521 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:43:47,930 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:43:47,930 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:43:47,930 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:43:48,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:43:49,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:50,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:50,368 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:43:52,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:54,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:56,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:43:58,654 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:43:58,654 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:44:07,967 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:44:07,968 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:44:07,968 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:44:08,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:44:09,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:10,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:12,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:13,785 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:44:13,785 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:44:14,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:16,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:20,438 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:44:28,025 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:44:28,026 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:44:28,026 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:44:28,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:44:28,924 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:44:28,925 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:44:29,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:30,309 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:32,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:34,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:36,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:44,089 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:44:44,089 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:44:48,117 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:44:48,117 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:44:48,118 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:44:48,317 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:44:49,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:50,318 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:50,505 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:44:52,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:54,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:56,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:44:59,222 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:44:59,223 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:45:08,169 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:45:08,170 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:45:08,170 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:45:08,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:45:09,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:10,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:12,327 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:14,328 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:14,357 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:45:14,357 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:45:16,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:20,581 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:45:28,239 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:45:28,239 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:45:28,240 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:45:28,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:45:29,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:29,501 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:45:29,501 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:45:30,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:32,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:34,337 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:36,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:44,652 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:45:44,652 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:45:48,280 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:45:48,280 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:45:48,280 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:45:48,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:45:49,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:50,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:50,648 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:45:52,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:54,345 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:56,346 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:45:59,785 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:45:59,786 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:46:08,323 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:46:08,323 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:46:08,324 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:46:08,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:46:09,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:10,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:12,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:14,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:14,918 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:46:14,918 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:46:16,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:20,716 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:46:28,389 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:46:28,389 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:46:28,389 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:46:29,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:29,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:46:30,058 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:46:30,059 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:46:30,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:32,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:34,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:36,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:45,221 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:46:45,222 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:46:48,447 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:46:48,447 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:46:48,448 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:46:49,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:46:50,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:50,791 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:46:52,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:54,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:46:56,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:00,353 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:47:00,354 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:47:08,494 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:47:08,495 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:47:08,495 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:47:09,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:47:10,376 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:12,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:14,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:15,485 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:47:15,485 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:47:16,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:20,864 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:47:28,549 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:47:28,549 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:47:28,550 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:47:29,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:47:30,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:30,647 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:47:30,648 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:47:32,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:34,386 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:36,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:45,795 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:47:45,796 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:47:48,607 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:47:48,607 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:47:48,607 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:47:49,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:47:50,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:50,938 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:47:51,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:53,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:55,395 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:47:57,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:00,926 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:48:00,926 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:48:08,668 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:48:08,668 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:48:08,669 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:48:09,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:48:10,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:11,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:13,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:15,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:16,061 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:48:16,062 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:48:17,405 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:21,012 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:48:28,757 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:48:28,758 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:48:28,758 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:48:29,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:48:30,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:31,200 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:48:31,201 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:48:31,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:33,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:35,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:37,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:46,358 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:48:46,359 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:48:48,836 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:48:48,837 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:48:48,837 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:48:49,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:48:50,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:51,083 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:48:51,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:53,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:55,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:48:57,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:01,494 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:49:01,494 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:49:08,905 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:49:08,905 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:49:08,905 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:49:09,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:49:10,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:11,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:13,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:15,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:16,627 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:49:16,628 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:49:17,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:21,160 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:49:28,981 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:49:28,981 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:49:28,982 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:49:29,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:49:30,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:31,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:31,776 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:49:31,776 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:49:33,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:35,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:37,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:46,916 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:49:46,916 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:49:49,050 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:49:49,050 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:49:49,051 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:49:49,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:49:50,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:51,232 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:49:51,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:53,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:55,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:49:57,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:02,049 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:50:02,050 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:50:09,127 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:50:09,127 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:50:09,128 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:50:09,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:50:10,454 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:11,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:13,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:15,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:17,181 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:50:17,181 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:50:17,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:21,302 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:50:29,212 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:50:29,212 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:50:29,212 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:50:29,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:50:30,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:31,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:32,368 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:50:32,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:50:33,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:35,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:37,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:47,535 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:50:47,536 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:50:49,251 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:50:49,252 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:50:49,252 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:50:49,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:50:50,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:51,374 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:50:51,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:53,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:55,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:50:57,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:02,668 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:51:02,668 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:51:09,294 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:51:09,295 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:51:09,295 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:51:09,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:51:10,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:11,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:13,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:15,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:17,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:17,800 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:51:17,801 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:51:21,447 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:51:29,336 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:51:29,336 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:51:29,337 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:51:29,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:51:30,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:31,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:32,932 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:51:32,933 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:51:33,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:35,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:37,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:48,111 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:51:48,111 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:51:49,431 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:51:49,431 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:51:49,432 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:51:49,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:51:51,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:51,521 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:51:53,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:55,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:51:57,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:03,242 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:52:03,242 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:52:09,470 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:52:09,471 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:52:09,471 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:52:09,507 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:52:11,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:13,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:15,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:17,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:18,376 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:52:18,377 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:52:21,592 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:52:29,510 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:52:29,510 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:52:29,511 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:52:29,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:52:31,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:33,512 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:52:33,512 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:52:33,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:35,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:37,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:48,645 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:52:48,645 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:52:49,547 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:52:49,548 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:52:49,548 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:52:50,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:52:51,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:51,668 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:52:53,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:55,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:52:57,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:03,784 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:53:03,784 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:53:09,584 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:53:09,584 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:53:09,584 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:53:10,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:53:11,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:13,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:15,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:17,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:18,940 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:53:18,941 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:53:21,742 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:53:29,676 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:53:29,676 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:53:29,677 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:53:30,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:53:31,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:33,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:34,074 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:53:34,074 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:53:35,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:37,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:49,214 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:53:49,214 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:53:49,739 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:53:49,740 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:53:49,740 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:53:50,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:53:51,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:51,811 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:53:53,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:55,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:53:57,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:04,345 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:54:04,345 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:54:09,797 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:54:09,797 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:54:09,798 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:54:10,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:54:11,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:12,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:14,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:16,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:18,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:19,476 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:54:19,476 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:54:21,881 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:54:29,848 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:54:29,849 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:54:29,849 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:54:30,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:54:31,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:32,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:34,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:34,679 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:54:34,680 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:54:36,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:38,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:49,842 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:54:49,842 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:54:50,015 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:54:50,016 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:54:50,016 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:54:50,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:54:51,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:51,952 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:54:52,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:54,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:56,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:54:58,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:05,055 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:55:05,055 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:55:10,068 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:55:10,068 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:55:10,069 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:55:10,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:55:11,587 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:12,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:14,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:16,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:18,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:20,189 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:55:20,189 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:55:22,023 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:55:30,123 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:55:30,123 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:55:30,124 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:55:30,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:55:31,596 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:32,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:34,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:35,318 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:55:35,319 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:55:36,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:38,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:50,194 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:55:50,194 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:55:50,195 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:55:50,466 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:55:50,467 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:55:50,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:55:51,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:52,093 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:55:52,605 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:54,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:56,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:55:58,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:05,599 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:56:05,599 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:56:10,230 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:56:10,231 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:56:10,231 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:56:10,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:56:11,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:12,614 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:14,615 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:16,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:18,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:20,737 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:56:20,738 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:56:22,163 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:56:30,298 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:56:30,298 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:56:30,299 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:56:30,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:56:31,622 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:32,623 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:34,624 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:35,877 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:56:35,878 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:56:36,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:38,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:50,360 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:56:50,360 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:56:50,361 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:56:50,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:56:51,041 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:56:51,042 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:56:51,632 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:52,237 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:56:52,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:54,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:56,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:56:58,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:06,179 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:57:06,179 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:57:10,402 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:57:10,403 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:57:10,403 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:57:10,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:57:11,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:12,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:14,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:16,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:18,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:21,310 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:57:21,310 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:57:22,307 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:57:30,487 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:57:30,488 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:57:30,488 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:57:30,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:57:31,650 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:32,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:34,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:36,441 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:57:36,442 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:57:36,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:38,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:50,537 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:57:50,538 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:57:50,539 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:57:50,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:57:51,609 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:57:51,609 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:57:51,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:52,379 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:57:52,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:54,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:56,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:57:58,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:06,813 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:58:06,813 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:58:10,684 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:58:10,684 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:58:10,685 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:58:11,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:58:12,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:14,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:16,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:18,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:21,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:58:21,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:58:22,451 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:58:30,742 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:58:30,743 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:58:30,743 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:58:31,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:58:32,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:34,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:36,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:37,077 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:58:37,078 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:58:38,682 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:50,814 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:58:50,814 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:58:50,815 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:58:51,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:58:52,239 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:58:52,239 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:58:52,523 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:58:52,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:54,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:56,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:58:58,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:07,395 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:59:07,396 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:59:10,889 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:59:10,890 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:59:10,890 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:59:11,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:59:12,696 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:15,697 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:17,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:19,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:22,531 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:59:22,531 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:59:22,661 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:59:30,936 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:59:30,936 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:59:30,936 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:59:31,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:59:32,705 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:33,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:35,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:37,663 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:59:37,663 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:59:37,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:39,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:50,998 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 08:59:50,998 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 08:59:50,999 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 08:59:51,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 08:59:52,667 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 08:59:52,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:52,807 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 08:59:52,808 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 08:59:53,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:55,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:57,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 08:59:59,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:07,942 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:00:07,942 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:00:11,060 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:00:11,061 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:00:11,061 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:00:11,722 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:00:12,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:13,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:15,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:17,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:19,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:22,742 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:00:23,075 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:00:23,075 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:00:31,131 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:00:31,131 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:00:31,132 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:00:31,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:00:32,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:33,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:35,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:37,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:38,204 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:00:38,205 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:00:39,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:51,174 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:00:51,174 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:00:51,175 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:00:51,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:00:52,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:52,813 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:00:53,361 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:00:53,361 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:00:53,742 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:55,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:57,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:00:59,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:08,742 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:01:08,742 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:01:11,231 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:01:11,231 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:01:11,232 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:01:11,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:01:12,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:13,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:15,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:17,753 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:19,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:22,884 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:01:23,890 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:01:23,890 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:01:31,271 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:01:31,271 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:01:31,272 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:01:31,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:01:32,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:33,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:35,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:37,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:39,020 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:01:39,020 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:01:39,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:51,310 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:01:51,310 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:01:51,311 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:01:51,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:01:52,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:52,957 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:01:53,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:54,169 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:01:54,169 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:01:55,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:57,771 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:01:59,772 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:09,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:02:09,323 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:02:11,351 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:02:11,352 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:02:11,352 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:02:11,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:02:12,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:13,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:15,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:17,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:19,780 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:23,026 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:02:24,454 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:02:24,455 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:02:31,414 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:02:31,414 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:02:31,415 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:02:31,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:02:32,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:33,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:35,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:37,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:39,583 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:02:39,584 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:02:39,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:51,484 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:02:51,484 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:02:51,484 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:02:51,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:02:52,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:53,102 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:02:53,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:54,717 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:02:54,717 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:02:55,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:57,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:02:59,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:09,847 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:03:09,848 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:03:11,524 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:03:11,525 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:03:11,525 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:03:11,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:03:12,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:13,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:15,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:17,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:19,806 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:23,174 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:03:24,981 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:03:24,981 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:03:31,583 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:03:31,583 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:03:31,584 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:03:31,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:03:33,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:35,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:37,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:39,815 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:40,111 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:03:40,111 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:03:51,636 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:03:51,636 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:03:51,637 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:03:51,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:03:53,246 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:03:53,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:55,249 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:03:55,249 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:03:55,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:57,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:03:59,824 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:01,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:03,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:05,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:07,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:09,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:10,383 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:04:10,383 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:04:11,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:13,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:15,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:17,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:19,833 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:21,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:23,322 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:04:24,835 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:25,514 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:04:25,514 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:04:26,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:28,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:30,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:32,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:34,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:36,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:38,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:40,669 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:04:40,669 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:04:40,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:42,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:44,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:46,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:48,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:50,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:52,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:04:53,400 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-17 09:04:55,494 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-17 09:04:55,495 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-17 09:04:55,495 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-17 09:04:55,804 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:04:55,805 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-17 09:04:55,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-17 09:05:04,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:05:07,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-17 09:05:10,945 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-17 09:05:10,945 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status