diff --git "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" --- "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" +++ "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" @@ -23046,3 +23046,3698 @@ 2021-07-16 20:34:29,647 DEBUG SenderThread:783720 [sender.py:send():179] send: stats 2021-07-16 20:34:33,234 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status 2021-07-16 20:34:33,235 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:34:48,367 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:34:48,368 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:34:59,728 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:35:03,796 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:35:03,797 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:35:18,929 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:35:18,930 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:35:29,805 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:35:34,072 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:35:34,072 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:35:45,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:35:47,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:35:49,203 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:35:49,203 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:35:49,374 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:35:51,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:35:53,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:35:59,883 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:36:03,762 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:36:03,762 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:36:03,763 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:36:04,370 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:36:04,370 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:36:04,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:36:04,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:05,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:07,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:09,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:11,383 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:13,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:19,508 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:36:19,509 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:36:23,800 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:36:23,800 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:36:23,801 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:36:24,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:36:24,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:25,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:27,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:29,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:29,960 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:36:31,391 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:34,638 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:36:34,638 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:36:43,836 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:36:43,837 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:36:43,837 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:36:44,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:36:44,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:45,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:47,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:49,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:36:49,775 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:36:49,776 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:36:51,399 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:00,038 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:37:03,892 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:37:03,892 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:37:03,893 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:37:04,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:37:04,949 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:37:04,949 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:37:05,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:06,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:07,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:09,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:11,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:20,115 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:37:20,115 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:37:23,989 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:37:23,989 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:37:23,989 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:37:24,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:37:24,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:25,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:27,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:29,415 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:30,111 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:37:31,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:33,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:35,245 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:37:35,245 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:37:44,043 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:37:44,043 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:37:44,044 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:37:44,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:37:44,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:45,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:47,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:49,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:50,389 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:37:50,389 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:37:51,424 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:37:53,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:00,182 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:38:04,143 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:38:04,143 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:38:04,144 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:38:04,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:38:05,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:05,553 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:38:05,553 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:38:07,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:09,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:11,432 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:13,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:20,697 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:38:20,698 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:38:24,182 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:38:24,182 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:38:24,182 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:38:24,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:38:25,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:27,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:29,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:30,251 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:38:31,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:33,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:35,831 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:38:35,832 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:38:44,261 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:38:44,261 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:38:44,262 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:38:44,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:38:45,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:47,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:48,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:50,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:38:50,971 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:38:50,971 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:38:52,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:00,320 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:39:04,305 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:39:04,306 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:39:04,306 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:39:04,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:39:05,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:06,103 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:39:06,103 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:39:06,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:08,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:10,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:12,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:21,241 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:39:21,242 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:39:24,394 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:39:24,395 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:39:24,395 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:39:24,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:39:25,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:26,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:28,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:30,392 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:39:30,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:32,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:36,372 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:39:36,372 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:39:44,513 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:39:44,514 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:39:44,514 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:39:45,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:39:45,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:46,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:48,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:50,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:39:51,502 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:39:51,502 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:39:52,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:00,467 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:40:04,597 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:40:04,597 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:40:04,598 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:40:05,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:40:05,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:06,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:06,641 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:40:06,642 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:40:08,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:10,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:12,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:14,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:21,798 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:40:21,799 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:40:24,634 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:40:24,635 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:40:24,635 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:40:25,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:40:25,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:26,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:28,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:30,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:30,541 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:40:32,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:36,929 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:40:36,930 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:40:44,685 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:40:44,686 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:40:44,686 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:40:45,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:40:45,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:46,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:48,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:50,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:40:52,063 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:40:52,063 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:40:52,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:00,619 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:41:04,746 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:41:04,746 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:41:04,747 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:41:05,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:41:06,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:07,207 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:41:07,208 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:41:07,508 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:08,508 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:10,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:12,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:22,370 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:41:22,371 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:41:24,784 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:41:24,785 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:41:24,785 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:41:25,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:41:25,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:26,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:28,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:30,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:30,699 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:41:32,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:37,505 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:41:37,505 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:41:44,831 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:41:44,831 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:41:44,832 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:41:45,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:41:45,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:46,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:48,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:50,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:52,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:41:52,638 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:41:52,639 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:42:00,776 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:42:04,869 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:42:04,870 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:42:04,870 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:42:05,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:42:05,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:06,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:07,781 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:42:07,781 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:42:08,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:10,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:12,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:23,113 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:42:23,114 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:42:24,997 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:42:24,997 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:42:24,998 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:42:25,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:42:25,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:26,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:28,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:30,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:30,854 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:42:32,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:38,244 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:42:38,245 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:42:45,062 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:42:45,063 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:42:45,063 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:42:45,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:42:46,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:48,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:50,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:52,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:42:53,380 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:42:53,380 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:43:00,931 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:43:05,118 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:43:05,119 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:43:05,119 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:43:05,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:43:06,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:08,517 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:43:08,517 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:43:08,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:10,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:12,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:23,667 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:43:23,668 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:43:25,159 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:43:25,160 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:43:25,160 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:43:25,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:43:26,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:29,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:31,009 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:43:31,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:33,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:38,801 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:43:38,802 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:43:45,217 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:43:45,218 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:43:45,218 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:43:45,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:43:46,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:47,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:49,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:51,581 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:53,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:43:53,932 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:43:53,933 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:44:01,087 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:44:05,255 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:44:05,255 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:44:05,256 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:44:05,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:44:06,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:07,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:09,064 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:44:09,065 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:44:09,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:11,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:13,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:24,195 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:44:24,195 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:44:25,294 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:44:25,294 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:44:25,295 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:44:25,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:44:26,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:27,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:29,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:31,165 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:44:31,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:33,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:39,326 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:44:39,327 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:44:45,334 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:44:45,334 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:44:45,335 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:44:45,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:44:46,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:47,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:49,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:51,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:53,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:44:54,460 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:44:54,460 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:45:01,242 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:45:05,390 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:45:05,390 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:45:05,391 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:45:05,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:45:06,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:07,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:09,592 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:45:09,593 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:45:09,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:11,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:13,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:24,731 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:45:24,731 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:45:25,473 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:45:25,474 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:45:25,474 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:45:25,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:45:26,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:27,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:29,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:31,321 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:45:31,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:33,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:39,863 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:45:39,863 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:45:45,522 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:45:45,522 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:45:45,522 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:45:45,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:45:46,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:47,632 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:49,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:51,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:53,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:45:54,996 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:45:54,996 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:46:01,398 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:46:05,581 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:46:05,581 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:46:05,582 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:46:05,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:46:06,640 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:07,640 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:09,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:10,128 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:46:10,129 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:46:11,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:13,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:25,265 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:46:25,265 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:46:25,646 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:46:25,646 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:46:25,647 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:46:25,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:46:26,648 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:27,649 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:29,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:31,476 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:46:31,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:33,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:40,395 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:46:40,396 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:46:45,717 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:46:45,717 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:46:45,718 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:46:46,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:46:46,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:47,657 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:49,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:51,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:53,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:46:55,534 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:46:55,534 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:47:01,560 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:47:05,812 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:47:05,812 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:47:05,813 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:47:06,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:47:06,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:07,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:09,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:10,664 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:47:10,665 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:47:11,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:13,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:25,799 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:47:25,799 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:47:25,934 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:47:25,934 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:47:25,934 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:47:26,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:47:26,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:27,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:29,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:31,639 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:47:31,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:33,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:40,946 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:47:40,947 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:47:45,925 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:47:45,925 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:47:45,926 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:47:46,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:47:47,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:49,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:51,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:53,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:47:56,079 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:47:56,079 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:48:01,717 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:48:05,984 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:48:05,985 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:48:05,985 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:48:06,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:48:07,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:09,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:11,218 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:48:11,218 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:48:11,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:13,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:26,041 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:48:26,041 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:48:26,042 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:48:26,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:48:26,355 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:48:26,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:48:27,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:29,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:31,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:31,793 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:48:33,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:41,515 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:48:41,516 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:48:46,130 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:48:46,131 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:48:46,131 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:48:46,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:48:47,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:49,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:51,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:53,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:48:56,649 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:48:56,649 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:49:01,873 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:49:06,187 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:49:06,187 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:49:06,188 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:49:06,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:49:07,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:09,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:11,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:11,782 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:49:11,782 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:49:13,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:26,263 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:49:26,264 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:49:26,264 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:49:26,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:49:26,959 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:49:26,959 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:49:27,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:30,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:31,952 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:49:32,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:34,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:42,090 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:49:42,091 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:49:46,326 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:49:46,326 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:49:46,327 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:49:46,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:49:47,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:48,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:50,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:52,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:54,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:49:57,221 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:49:57,221 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:50:02,029 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:50:06,433 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:50:06,433 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:50:06,434 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:50:06,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:50:07,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:08,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:10,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:12,351 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:50:12,351 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:50:12,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:14,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:26,509 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:50:26,509 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:50:26,510 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:50:26,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:50:27,491 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:50:27,492 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:50:27,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:28,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:30,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:32,108 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:50:32,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:34,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:42,740 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:50:42,740 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:50:46,549 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:50:46,549 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:50:46,550 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:50:46,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:50:47,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:48,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:50,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:52,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:54,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:50:57,870 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:50:57,870 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:51:02,186 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:51:06,588 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:51:06,589 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:51:06,589 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:51:06,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:51:07,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:08,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:10,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:12,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:13,001 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:51:13,001 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:51:14,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:26,625 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:51:26,626 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:51:26,626 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:51:26,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:51:27,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:28,190 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:51:28,191 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:51:28,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:30,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:32,262 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:51:32,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:34,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:43,320 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:51:43,320 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:51:46,687 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:51:46,688 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:51:46,688 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:51:46,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:51:47,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:48,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:50,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:52,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:54,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:51:58,452 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:51:58,453 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:52:02,340 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:52:06,727 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:52:06,728 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:52:06,728 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:52:06,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:52:07,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:08,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:10,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:12,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:13,584 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:52:13,584 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:52:14,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:26,813 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:52:26,813 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:52:26,814 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:52:27,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:52:28,719 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:52:28,720 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:52:28,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:30,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:32,418 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:52:32,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:34,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:43,866 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:52:43,866 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:52:46,851 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:52:46,851 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:52:46,852 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:52:47,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:52:48,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:50,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:52,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:54,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:52:58,997 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:52:58,998 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:53:02,495 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:53:06,930 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:53:06,931 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:53:06,931 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:53:07,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:53:08,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:10,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:12,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:14,131 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:53:14,131 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:53:14,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:27,004 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:53:27,005 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:53:27,005 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:53:27,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:53:28,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:29,370 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:53:29,370 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:53:30,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:32,573 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:53:32,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:34,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:44,525 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:53:44,525 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:53:47,055 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:53:47,055 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:53:47,056 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:53:47,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:53:48,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:50,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:52,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:54,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:53:59,657 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:53:59,657 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:54:02,651 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:54:07,106 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:54:07,106 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:54:07,107 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:54:07,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:54:08,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:10,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:12,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:14,790 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:54:14,790 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:54:14,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:27,189 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:54:27,190 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:54:27,190 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:54:27,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:54:28,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:29,997 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:54:29,997 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:54:30,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:32,729 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:54:32,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:34,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:45,176 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:54:45,176 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:54:47,229 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:54:47,229 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:54:47,230 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:54:47,865 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:54:48,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:51,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:53,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:54:55,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:00,308 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:55:00,309 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:55:02,808 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:55:07,310 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:55:07,311 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:55:07,311 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:55:07,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:55:08,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:09,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:11,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:13,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:15,440 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:55:15,440 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:55:15,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:27,407 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:55:27,407 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:55:27,408 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:55:27,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:55:28,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:29,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:30,580 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:55:30,580 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:55:31,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:32,886 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:55:33,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:35,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:45,734 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:55:45,734 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:55:47,446 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:55:47,446 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:55:47,447 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:55:47,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:55:48,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:49,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:51,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:53,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:55,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:57,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:55:59,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:00,872 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:56:00,873 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:56:01,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:02,963 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:56:03,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:05,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:07,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:09,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:11,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:13,908 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:15,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:16,004 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:56:16,004 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:56:17,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:19,911 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:21,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:23,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:25,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:27,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:29,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:31,140 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:56:31,140 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:56:31,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:33,040 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:56:33,917 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:35,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:37,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:39,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:41,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:43,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:45,922 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:46,271 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:56:46,271 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:56:47,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:51,317 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:56:51,317 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:56:51,318 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:56:51,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:56:52,925 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:54,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:56,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:56:58,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:00,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:01,403 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:57:01,404 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:57:03,118 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:57:11,406 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:57:11,406 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:57:11,407 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:57:11,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:57:12,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:14,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:16,537 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:57:16,537 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:57:16,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:18,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:20,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:31,452 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:57:31,453 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:57:31,453 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:57:31,794 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:57:31,794 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:57:31,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:57:32,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:33,195 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:57:34,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:36,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:38,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:40,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:46,961 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:57:46,961 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:57:51,508 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:57:51,509 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:57:51,509 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:57:51,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:57:52,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:53,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:54,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:56,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:57:58,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:00,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:02,093 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:58:02,093 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:58:03,272 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:58:11,607 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:58:11,608 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:58:11,608 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:58:11,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:58:12,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:13,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:14,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:16,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:17,226 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:58:17,226 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:58:18,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:20,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:31,646 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:58:31,646 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:58:31,647 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:58:31,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:58:32,365 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:58:32,365 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:58:32,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:33,350 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:58:33,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:34,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:36,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:38,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:40,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:47,502 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:58:47,502 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:58:51,742 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:58:51,743 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:58:51,743 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:58:51,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:58:52,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:53,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:54,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:56,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:58:58,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:00,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:02,636 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:59:02,637 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:59:03,424 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:59:11,821 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:59:11,822 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:59:11,822 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:59:11,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:59:12,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:13,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:14,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:16,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:17,771 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:59:17,771 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:59:18,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:20,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:31,908 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:59:31,909 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:59:31,909 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:59:31,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:59:32,913 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:59:32,914 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:59:32,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:33,495 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 20:59:33,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:34,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:36,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:38,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:40,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:48,091 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 20:59:48,091 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 20:59:52,002 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 20:59:52,003 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 20:59:52,003 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 20:59:52,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 20:59:53,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:54,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:55,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:57,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 20:59:59,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:01,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:03,233 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:00:03,233 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:00:03,575 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:00:12,082 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:00:12,083 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:00:12,083 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:00:13,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:00:13,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:14,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:15,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:17,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:18,368 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:00:18,368 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:00:19,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:21,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:32,156 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:00:32,157 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:00:32,157 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:00:33,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:00:33,502 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:00:33,503 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:00:33,652 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:00:34,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:35,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:37,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:39,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:41,027 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:48,649 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:00:48,650 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:00:52,196 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:00:52,196 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:00:52,196 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:00:53,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:00:53,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:54,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:55,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:57,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:00:59,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:01,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:03,729 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:01:03,780 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:01:03,780 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:01:12,294 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:01:12,295 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:01:12,295 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:01:13,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:01:13,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:14,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:15,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:17,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:18,908 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:01:18,909 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:01:19,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:21,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:32,366 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:01:32,367 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:01:32,367 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:01:33,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:01:33,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:33,807 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:01:34,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:34,054 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:01:34,054 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:01:35,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:37,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:39,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:41,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:49,233 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:01:49,234 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:01:52,405 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:01:52,405 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:01:52,406 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:01:53,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:01:53,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:54,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:55,057 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:57,057 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:01:59,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:01,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:03,883 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:02:04,367 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:02:04,368 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:02:12,528 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:02:12,528 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:02:12,529 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:02:13,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:02:14,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:16,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:18,065 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:19,499 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:02:19,500 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:02:20,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:22,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:32,567 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:02:32,567 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:02:32,568 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:02:33,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:02:33,963 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:02:34,072 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:34,635 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:02:34,636 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:02:36,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:38,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:40,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:42,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:49,808 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:02:49,809 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:02:52,633 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:02:52,633 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:02:52,634 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:02:53,080 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:02:54,081 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:56,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:02:58,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:00,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:02,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:04,041 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:03:04,941 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:03:04,942 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:03:12,670 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:03:12,671 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:03:12,671 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:03:13,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:03:14,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:16,089 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:18,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:20,077 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:03:20,077 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:03:20,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:32,772 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:03:32,773 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:03:32,773 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:03:33,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:03:34,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:34,122 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:03:35,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:35,211 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:03:35,212 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:03:36,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:38,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:40,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:50,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:03:50,354 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:03:52,848 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:03:52,848 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:03:52,849 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:03:53,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:03:54,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:55,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:56,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:03:58,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:00,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:02,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:04,201 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:04:05,484 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:04:05,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:04:12,903 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:04:12,904 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:04:12,904 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:04:13,112 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:04:14,112 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:15,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:16,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:18,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:20,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:20,614 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:04:20,614 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:04:22,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:32,951 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:04:32,951 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:04:32,952 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:04:33,120 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:04:34,279 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:04:35,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:35,783 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:04:35,783 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:04:37,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:39,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:41,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:43,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:50,982 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:04:50,982 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:04:52,989 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:04:52,989 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:04:52,990 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:04:53,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:04:55,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:57,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:04:59,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:01,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:04,357 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:05:06,118 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:05:06,119 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:05:13,064 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:05:13,064 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:05:13,064 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:05:13,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:05:15,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:17,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:19,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:21,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:21,251 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:05:21,251 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:05:33,157 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:05:33,157 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:05:33,158 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:05:34,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:05:34,434 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:05:35,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:36,409 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:05:36,409 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:05:37,147 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:39,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:41,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:51,545 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:05:51,545 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:05:53,193 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:05:53,194 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:05:53,195 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:05:54,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:05:55,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:57,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:05:59,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:01,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:04,510 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:06:06,676 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:06:06,677 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:06:13,257 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:06:13,257 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:06:13,258 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:06:14,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:06:15,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:17,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:19,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:21,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:21,811 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:06:21,811 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:06:33,296 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:06:33,297 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:06:33,297 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:06:34,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:06:34,588 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:06:35,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:36,944 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:06:36,945 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:06:37,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:39,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:41,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:52,079 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:06:52,079 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:06:53,362 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:06:53,362 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:06:53,363 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:06:54,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:06:55,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:57,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:06:59,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:01,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:04,667 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:07:07,211 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:07:07,211 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:07:13,409 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:07:13,410 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:07:13,410 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:07:14,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:07:15,187 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:17,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:19,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:21,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:22,342 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:07:22,342 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:07:33,486 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:07:33,486 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:07:33,487 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:07:34,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:07:34,744 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:07:35,195 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:37,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:37,470 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:07:37,471 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:07:39,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:41,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:52,686 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:07:52,687 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:07:53,567 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:07:53,568 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:07:53,568 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:07:54,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:07:55,203 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:57,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:07:59,205 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:01,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:04,822 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:08:07,818 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:08:07,819 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:08:13,607 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:08:13,607 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:08:13,607 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:08:14,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:08:15,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:17,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:19,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:21,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:22,950 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:08:22,950 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:08:33,707 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:08:33,708 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:08:33,708 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:08:34,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:08:34,900 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:08:35,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:37,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:38,547 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:08:38,547 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:08:39,221 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:41,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:53,720 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:08:53,721 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:08:53,854 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:08:53,854 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:08:53,855 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:08:54,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:08:55,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:56,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:08:57,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:00,230 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:02,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:04,977 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:09:08,856 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:09:08,856 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:09:13,828 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:09:13,829 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:09:13,829 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:09:14,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:09:16,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:18,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:20,238 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:22,239 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:23,984 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:09:23,984 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:09:33,885 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:09:33,885 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:09:33,885 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:09:34,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:09:35,051 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:09:36,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:38,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:39,114 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:09:39,115 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:09:40,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:42,247 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:53,965 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:09:53,966 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:09:53,966 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:09:54,252 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:09:54,271 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:09:54,272 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:09:56,253 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:09:58,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:00,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:02,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:05,127 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:10:09,419 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:10:09,419 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:10:14,025 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:10:14,026 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:10:14,026 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:10:14,261 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:10:16,262 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:18,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:20,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:22,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:24,551 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:10:24,551 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:10:34,071 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:10:34,071 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:10:34,072 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:10:34,270 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:10:35,203 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:10:36,271 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:38,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:39,683 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:10:39,684 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:10:40,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:42,273 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:54,179 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:10:54,179 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:10:54,179 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:10:54,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:10:54,947 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:10:54,947 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:10:56,279 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:10:58,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:00,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:02,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:05,275 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:11:10,097 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:11:10,097 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:11:14,236 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:11:14,237 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:11:14,237 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:11:14,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:11:16,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:18,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:20,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:22,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:25,237 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:11:25,237 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:11:34,294 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:11:34,295 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:11:34,295 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:11:34,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:11:35,344 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:11:36,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:38,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:40,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:40,446 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:11:40,446 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:11:42,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:54,342 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:11:54,342 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:11:54,343 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:11:55,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:11:55,582 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:11:55,583 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:11:56,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:11:58,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:00,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:02,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:05,414 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:12:10,717 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:12:10,717 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:12:14,428 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:12:14,428 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:12:14,429 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:12:15,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:12:16,314 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:18,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:20,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:22,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:25,847 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:12:25,848 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:12:34,482 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:12:34,483 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:12:34,483 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:12:35,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:12:35,482 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:12:36,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:38,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:40,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:41,019 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:12:41,019 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:12:42,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:54,560 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:12:54,560 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:12:54,561 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:12:55,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:12:56,156 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:12:56,157 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:12:56,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:57,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:12:59,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:01,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:03,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:05,552 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:13:11,319 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:13:11,319 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:13:14,664 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:13:14,664 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:13:14,665 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:13:15,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:13:16,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:17,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:19,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:21,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:23,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:26,451 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:13:26,451 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:13:34,721 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:13:34,721 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:13:34,722 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:13:35,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:13:35,623 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:13:36,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:37,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:39,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:41,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:41,583 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:13:41,583 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:13:43,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:54,856 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:13:54,856 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:13:54,857 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:13:55,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:13:56,754 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:13:56,755 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:13:57,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:13:59,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:01,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:03,356 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:05,695 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:14:11,894 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:14:11,895 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:14:14,892 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:14:14,893 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:14:14,893 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:14:15,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:14:17,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:19,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:21,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:23,365 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:27,026 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:14:27,027 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:14:35,011 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:14:35,012 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:14:35,012 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:14:35,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:14:35,768 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:14:37,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:39,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:41,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:42,163 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:14:42,163 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:14:43,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:55,066 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:14:55,066 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:14:55,067 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:14:55,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:14:57,294 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:14:57,295 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:14:57,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:14:59,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:01,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:03,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:05,844 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:15:12,437 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:15:12,438 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:15:15,117 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:15:15,118 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:15:15,118 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:15:15,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:15:17,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:19,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:21,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:23,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:27,569 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:15:27,569 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:15:35,171 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:15:35,171 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:15:35,171 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:15:35,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:15:35,923 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:15:37,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:39,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:41,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:42,702 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:15:42,702 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:15:43,399 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:55,220 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:15:55,221 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:15:55,221 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:15:55,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:15:57,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:15:57,838 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:15:57,838 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:15:59,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:01,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:03,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:06,000 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:16:13,155 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:16:13,155 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:16:15,260 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:16:15,261 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:16:15,261 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:16:15,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:16:17,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:19,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:21,415 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:23,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:28,287 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:16:28,288 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:16:35,328 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:16:35,328 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:16:35,328 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:16:35,420 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:16:36,078 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:16:37,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:39,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:41,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:43,417 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:16:43,418 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:16:43,424 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:55,394 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:16:55,395 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:16:55,395 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:16:55,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:16:57,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:16:58,566 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:16:58,566 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:16:59,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:01,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:03,432 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:06,155 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:17:13,750 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:17:13,750 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:17:15,470 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:17:15,470 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:17:15,471 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:17:16,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:17:17,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:19,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:21,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:23,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:28,885 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:17:28,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:17:35,552 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:17:35,553 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:17:35,553 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:17:36,234 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:17:36,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:17:37,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:39,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:41,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:43,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:44,015 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:17:44,015 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:17:55,654 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:17:55,655 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:17:55,655 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:17:56,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:17:57,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:17:59,157 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:17:59,158 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:17:59,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:01,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:03,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:06,311 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:18:14,292 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:18:14,292 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:18:15,720 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:18:15,720 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:18:15,721 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:18:16,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:18:17,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:19,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:22,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:24,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:29,435 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:18:29,436 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:18:35,812 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:18:35,812 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:18:35,813 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:18:36,386 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:18:36,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:18:37,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:38,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:40,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:42,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:44,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:44,567 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:18:44,568 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:18:55,870 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:18:55,870 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:18:55,871 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:18:56,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:18:57,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:58,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:18:59,733 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:18:59,733 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:19:00,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:02,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:04,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:06,466 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:19:14,867 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:19:14,868 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:19:15,925 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:19:15,926 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:19:15,926 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:19:16,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:19:17,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:18,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:20,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:22,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:24,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:29,999 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:19:29,999 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:19:36,023 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:19:36,023 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:19:36,023 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:19:36,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:19:36,543 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:19:37,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:38,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:40,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:42,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:44,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:45,131 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:19:45,131 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:19:56,062 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:19:56,063 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:19:56,063 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:19:56,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:19:57,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:19:58,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:00,262 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:20:00,262 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:20:00,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:02,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:04,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:06,620 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:20:15,394 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:20:15,394 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:20:16,139 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:20:16,140 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:20:16,140 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:20:16,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:20:17,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:18,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:20,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:22,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:24,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:30,524 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:20:30,524 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:20:36,180 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:20:36,180 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:20:36,181 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:20:36,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:20:36,696 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:20:37,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:38,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:40,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:42,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:44,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:45,655 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:20:45,656 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:20:56,238 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:20:56,239 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:20:56,239 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:20:56,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:20:57,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:20:58,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:00,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:00,836 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:21:00,836 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:21:02,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:04,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:06,772 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:21:15,966 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:21:15,966 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:21:16,289 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:21:16,290 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:21:16,290 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:21:16,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:21:18,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:20,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:22,540 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:24,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:31,099 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:21:31,099 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:21:36,326 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:21:36,327 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:21:36,327 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:21:36,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:21:36,851 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:21:38,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:40,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:42,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:44,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:21:46,229 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:21:46,229 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:21:56,376 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:21:56,376 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:21:56,377 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:21:56,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:21:58,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:00,557 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:01,363 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:22:01,363 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:22:02,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:04,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:06,929 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:22:16,434 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:22:16,434 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:22:16,435 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:22:16,493 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:22:16,494 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:22:16,564 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:22:18,565 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:20,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:22,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:24,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:31,666 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:22:31,667 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:22:36,491 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:22:36,491 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:22:36,492 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:22:36,573 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:22:37,006 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:22:38,574 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:40,575 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:42,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:44,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:22:46,797 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:22:46,797 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:22:56,573 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:22:56,573 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:22:56,574 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:22:56,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:22:58,583 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:00,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:01,928 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:23:01,928 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:23:02,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:04,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:07,083 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:23:16,626 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:23:16,626 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:23:16,627 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:23:17,059 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:23:17,060 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:23:17,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:23:18,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:20,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:22,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:24,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:32,225 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:23:32,225 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:23:36,664 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:23:36,665 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:23:36,665 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:23:37,161 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:23:37,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:23:38,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:41,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:43,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:45,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:47,357 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:23:47,358 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:23:56,778 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:23:56,779 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:23:56,779 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:23:57,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:23:58,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:23:59,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:01,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:02,491 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:24:02,492 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:24:03,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:05,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:07,237 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:24:16,829 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:24:16,830 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:24:16,830 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:24:17,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:24:17,629 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:24:17,630 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:24:18,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:19,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:21,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:23,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:25,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:32,775 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:24:32,776 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:24:36,904 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:24:36,904 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:24:36,905 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:24:37,317 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:24:37,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:24:38,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:39,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:41,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:43,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:45,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:47,910 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:24:47,910 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:24:57,006 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:24:57,006 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:24:57,007 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:24:57,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:24:58,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:24:59,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:01,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:03,040 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:25:03,040 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:25:03,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:05,637 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:07,395 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:25:17,060 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:25:17,060 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:25:17,061 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:25:17,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:25:18,173 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:25:18,174 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:25:18,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:19,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:21,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:23,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:25,646 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:33,313 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:25:33,314 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:25:37,114 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:25:37,114 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:25:37,115 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:25:37,472 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:25:37,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:25:38,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:39,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:41,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:43,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:45,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:48,444 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:25:48,444 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:25:57,173 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:25:57,173 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:25:57,174 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:25:57,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:25:58,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:25:59,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:01,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:03,576 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:26:03,576 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:26:03,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:05,663 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:07,550 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:26:17,213 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:26:17,214 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:26:17,214 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:26:17,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:26:18,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:18,713 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:26:18,713 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:26:19,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:21,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:23,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:25,671 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:33,864 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:26:33,864 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:26:37,319 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:26:37,319 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:26:37,319 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:26:37,628 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:26:37,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:26:38,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:39,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:41,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:43,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:45,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:48,994 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:26:48,995 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:26:57,387 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:26:57,387 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:26:57,387 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:26:57,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:26:58,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:26:59,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:01,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:03,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:04,126 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:27:04,126 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:27:05,688 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:07,706 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:27:17,426 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:27:17,427 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:27:17,427 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:27:17,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:27:19,347 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:27:19,347 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:27:19,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:21,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:23,696 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:25,697 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:34,486 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:27:34,486 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:27:37,491 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:27:37,492 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:27:37,492 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:27:37,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:27:37,783 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:27:39,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:41,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:43,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:45,705 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:27:49,618 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:27:49,618 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:27:57,574 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:27:57,574 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:27:57,575 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:27:57,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:27:59,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:01,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:03,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:04,747 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:28:04,747 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:28:05,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:07,859 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:28:17,642 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:28:17,642 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:28:17,643 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:28:17,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:28:19,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:19,882 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:28:19,883 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:28:21,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:23,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:25,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:35,027 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:28:35,028 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:28:37,696 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:28:37,696 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:28:37,696 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:28:37,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:28:37,939 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:28:39,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:41,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:43,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:45,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:28:50,159 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:28:50,159 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:28:57,735 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:28:57,736 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:28:57,736 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:28:58,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:28:59,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:01,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:03,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:05,291 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:29:05,291 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:29:05,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:08,018 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:29:17,796 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:29:17,796 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:29:17,797 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:29:18,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:29:19,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:20,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:20,760 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:29:20,760 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:29:22,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:24,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:26,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:35,901 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:29:35,902 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:29:37,841 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:29:37,841 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:29:37,842 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:29:38,095 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:29:38,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:29:39,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:40,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:42,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:44,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:46,757 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:29:51,035 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:29:51,035 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:29:57,996 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:29:57,997 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:29:57,997 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:29:58,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:29:59,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:00,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:02,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:04,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:06,170 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:30:06,170 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:30:06,765 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:08,172 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:30:18,075 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:30:18,075 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:30:18,076 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:30:18,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:30:19,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:20,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:21,375 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:30:21,376 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:30:22,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:24,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:26,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:36,527 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:30:36,528 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:30:38,114 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:30:38,114 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:30:38,115 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:30:38,249 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:30:38,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:30:39,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:40,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:42,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:44,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:46,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:30:51,657 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:30:51,658 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:30:58,159 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:30:58,160 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:30:58,160 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:30:58,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:30:59,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:00,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:02,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:04,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:06,790 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:31:06,791 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:31:06,793 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:08,328 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:31:18,209 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:31:18,209 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:31:18,210 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:31:18,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:31:19,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:20,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:21,921 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:31:21,922 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:31:22,800 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:24,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:26,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:37,050 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:31:37,050 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:31:38,246 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:31:38,247 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:31:38,247 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:31:38,404 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:31:38,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:31:39,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:40,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:42,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:44,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:46,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:31:52,182 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:31:52,182 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:31:58,285 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:31:58,285 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:31:58,286 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:31:58,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:31:59,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:00,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:02,818 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:04,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:06,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:07,315 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:32:07,315 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:32:08,483 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:32:18,363 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:32:18,364 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:32:18,364 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:32:18,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:32:19,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:20,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:22,469 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:32:22,470 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:32:22,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:24,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:26,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:37,599 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:32:37,600 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:32:38,417 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:32:38,418 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:32:38,418 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:32:38,560 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:32:38,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:32:39,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:40,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:42,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:44,836 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:46,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:32:52,732 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:32:52,732 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:32:58,475 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:32:58,475 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:32:58,475 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:32:58,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:33:00,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:02,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:04,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:06,845 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:07,862 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:33:07,862 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:33:08,639 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:33:18,542 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:33:18,543 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:33:18,543 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:33:18,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:33:20,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:22,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:23,054 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:33:23,055 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:33:24,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:26,854 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:38,186 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:33:38,186 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:33:38,581 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:33:38,581 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:33:38,582 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:33:38,716 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:33:38,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:33:40,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:42,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:44,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:46,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:33:53,317 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:33:53,318 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:33:58,642 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:33:58,643 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:33:58,643 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:33:58,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:34:00,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:02,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:04,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:06,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:08,450 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:34:08,451 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:34:08,796 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:34:18,707 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:34:18,707 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:34:18,708 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:34:18,876 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:34:20,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:22,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:23,599 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:34:23,600 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:34:24,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:26,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:38,737 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:34:38,737 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:34:38,868 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:34:38,868 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:34:38,870 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:34:38,876 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:34:38,884 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:34:40,885 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:42,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:44,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:46,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:34:53,869 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:34:53,870 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:34:58,785 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:34:58,786 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:34:58,786 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:34:58,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:35:00,893 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:02,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:04,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:06,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:08,955 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:35:09,015 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:35:09,016 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:35:18,836 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:35:18,836 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:35:18,837 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:35:18,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:35:20,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:22,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:24,147 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:35:24,147 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:35:24,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:26,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:38,883 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:35:38,884 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:35:38,884 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:35:38,909 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:35:39,030 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:35:39,283 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:35:39,283 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:35:40,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:41,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:43,911 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:45,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:47,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:35:54,436 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:35:54,436 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:35:58,976 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:35:58,976 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:35:58,977 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:35:59,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:36:00,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:01,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:03,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:05,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:07,922 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:09,109 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:36:09,572 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:36:09,573 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:36:19,059 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:36:19,060 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:36:19,060 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:36:19,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:36:20,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:21,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:23,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:24,704 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:36:24,705 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:36:25,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:27,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:39,098 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:36:39,098 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:36:39,099 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:36:39,187 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:36:39,853 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:36:39,853 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:36:39,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:36:40,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:41,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:43,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:45,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:47,939 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:36:55,018 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:36:55,018 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:36:59,177 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:36:59,178 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:36:59,178 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:36:59,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:37:00,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:01,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:03,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:05,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:07,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:09,265 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:37:09,949 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:10,148 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:37:10,148 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:37:11,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:13,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:15,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:17,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:19,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:21,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:23,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:25,314 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:37:25,314 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:37:25,956 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:27,957 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:29,958 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:31,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:33,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:35,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:37,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:39,338 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:37:39,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:40,448 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:37:40,449 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:37:41,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:43,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:45,965 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:47,966 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:49,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:51,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:53,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:55,577 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:37:55,577 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:37:55,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:57,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:37:59,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:03,037 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:38:03,038 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:38:03,038 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:38:03,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:38:03,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:06,974 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:08,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:09,408 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:38:10,708 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:38:10,709 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:38:10,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:12,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:23,101 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:38:23,102 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:38:23,102 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:38:23,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:38:24,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:25,907 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:38:25,907 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:38:25,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:26,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:28,983 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:30,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:32,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:39,482 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:38:41,100 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:38:41,100 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:38:43,141 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:38:43,142 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:38:43,142 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:38:43,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:38:44,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:45,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:46,991 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:48,991 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:50,992 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:52,993 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:38:56,233 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:38:56,234 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:39:03,183 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:39:03,183 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:39:03,184 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:39:03,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:39:04,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:05,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:06,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:08,999 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:09,561 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:39:11,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:11,364 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:39:11,365 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:39:23,225 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:39:23,226 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:39:23,226 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:39:24,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:39:25,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:26,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:26,515 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:39:26,515 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:39:27,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:29,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:31,009 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:33,010 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:39,638 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:39:41,677 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:39:41,677 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:39:43,281 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:39:43,282 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:39:43,282 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:39:44,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:39:45,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:46,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:47,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:49,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:51,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:53,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:39:56,805 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:39:56,805 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:40:03,322 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:40:03,322 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:40:03,323 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:40:04,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:40:05,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:06,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:07,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:09,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:09,716 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:40:11,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:11,938 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:40:11,939 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:40:23,362 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:40:23,363 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:40:23,363 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:40:24,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:40:25,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:26,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:27,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:27,121 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:40:27,121 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:40:29,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:31,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:39,794 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:40:42,253 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:40:42,254 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:40:43,433 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:40:43,433 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:40:43,434 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:40:44,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:40:45,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:46,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:47,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:49,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:51,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:40:57,386 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:40:57,386 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:41:03,483 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:41:03,484 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:41:03,484 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:41:04,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:41:05,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:06,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:07,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:09,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:09,871 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:41:11,053 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:12,518 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:41:12,518 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:41:13,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:23,557 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:41:23,558 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:41:23,558 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:41:24,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:41:25,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:26,060 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:27,060 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:27,742 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:41:27,743 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:41:29,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:31,062 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:33,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:39,949 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:41:42,873 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:41:42,874 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:41:43,612 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:41:43,612 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:41:43,613 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:41:44,067 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:41:45,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:46,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:47,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:49,070 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:51,071 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:41:58,003 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:41:58,004 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:42:03,657 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:42:03,658 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:42:03,658 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:42:04,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:42:05,077 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:06,077 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:07,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:09,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:10,028 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:42:11,079 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:13,134 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:42:13,134 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:42:23,702 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:42:23,702 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:42:23,703 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:42:24,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:42:25,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:26,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:27,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:28,269 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:42:28,269 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:42:29,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:31,088 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:40,105 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:42:43,400 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:42:43,400 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:42:43,743 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:42:43,743 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:42:43,744 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:42:44,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:42:46,094 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:48,095 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:50,096 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:52,097 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:42:58,531 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:42:58,531 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:43:03,794 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:43:03,795 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:43:03,795 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:43:04,102 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:43:06,103 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:08,104 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:10,105 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:10,182 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:43:12,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:13,662 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:43:13,663 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:43:23,852 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:43:23,852 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:43:23,852 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:43:24,111 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:43:26,112 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:28,113 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:28,792 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:43:28,793 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:43:30,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:32,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:40,258 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:43:43,909 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:43:43,910 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:43:43,910 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:43:44,000 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:43:44,001 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:43:44,119 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:43:46,120 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:48,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:50,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:52,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:43:59,129 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:43:59,130 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:44:04,023 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:44:04,024 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:44:04,024 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:44:04,128 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:44:06,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:08,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:10,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:10,333 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:44:12,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:14,263 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:44:14,263 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:44:24,064 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:44:24,064 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:44:24,065 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:44:24,136 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:44:26,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:28,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:29,407 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:44:29,407 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:44:30,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:32,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:40,411 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:44:44,119 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:44:44,119 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:44:44,120 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:44:44,145 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:44:44,561 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:44:44,562 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:44:46,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:48,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:50,147 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:52,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:44:59,697 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:44:59,697 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:45:04,160 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:45:04,160 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:45:04,160 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:45:05,153 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:45:06,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:08,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:10,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:10,489 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:45:12,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:14,827 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:45:14,827 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:45:24,203 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:45:24,203 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:45:24,204 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:45:25,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:45:26,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:28,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:29,980 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:45:29,981 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:45:30,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:32,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:40,567 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:45:44,270 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:45:44,271 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:45:44,271 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:45:45,170 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:45:46,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:46,292 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:45:46,293 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:45:48,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:50,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:45:52,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:01,432 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:46:01,432 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:46:04,327 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:46:04,328 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:46:04,328 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:46:05,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:46:06,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:08,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:10,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:10,643 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:46:12,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:16,563 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:46:16,564 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:46:24,410 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:46:24,410 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:46:24,411 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:46:25,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:46:26,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:28,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:30,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:31,695 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:46:31,695 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:46:32,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:40,722 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:46:44,451 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:46:44,452 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:46:44,452 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:46:45,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:46:46,196 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:46,854 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:46:46,855 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:46:47,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:48,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:50,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:46:52,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:02,285 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:47:02,285 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:47:04,536 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:47:04,536 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:47:04,537 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:47:05,204 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:47:06,205 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:07,205 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:08,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:10,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:10,800 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:47:12,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:17,418 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:47:17,418 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:47:24,595 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:47:24,596 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:47:24,596 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:47:25,212 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:47:26,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:27,213 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:28,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:30,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:32,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:32,551 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:47:32,551 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:47:40,878 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:47:44,697 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:47:44,697 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:47:44,698 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:47:45,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:47:46,221 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:47,221 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:47,688 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:47:47,689 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:47:48,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:50,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:47:52,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:02,820 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:48:02,820 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:48:04,748 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:48:04,748 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:48:04,749 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:48:05,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:48:07,229 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:09,230 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:10,955 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:48:11,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:13,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:17,958 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:48:17,959 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:48:24,801 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:48:24,802 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:48:24,802 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:48:25,237 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:48:27,238 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:29,239 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:31,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:33,089 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:48:33,089 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:48:33,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:41,034 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:48:44,875 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:48:44,876 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:48:44,876 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:48:45,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:48:47,247 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:48,255 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:48:48,255 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:48:49,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:51,249 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:48:53,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:03,395 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:49:03,395 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:49:04,966 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:49:04,966 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:49:04,967 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:49:05,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:49:07,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:09,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:11,111 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:49:11,258 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:13,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:18,530 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:49:18,531 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:49:25,008 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:49:25,008 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:49:25,008 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:49:25,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:49:27,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:29,266 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:31,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:33,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:33,661 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:49:33,661 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:49:41,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:49:45,049 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:49:45,050 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:49:45,050 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:49:45,273 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:49:47,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:48,908 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:49:48,908 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:49:49,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:51,275 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:49:53,276 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:04,037 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:50:04,038 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:50:05,090 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:50:05,091 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:50:05,091 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:50:05,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:50:07,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:09,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:11,266 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:50:11,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:13,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:19,169 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:50:19,169 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:50:25,158 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:50:25,158 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:50:25,159 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:50:25,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:50:27,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:29,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:31,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:33,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:34,305 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:50:34,306 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:50:41,343 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:50:45,217 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:50:45,218 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:50:45,218 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:50:45,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:50:47,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:49,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:49,436 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:50:49,436 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:50:51,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:50:53,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:04,569 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:51:04,569 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:51:05,288 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:51:05,288 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:51:05,289 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:51:05,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:51:07,310 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:09,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:11,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:11,421 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:51:13,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:19,704 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:51:19,704 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:51:25,328 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:51:25,329 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:51:25,330 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:51:26,319 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:51:27,319 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:29,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:31,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:33,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:34,833 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:51:34,834 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:51:41,498 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:51:45,433 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:51:45,433 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:51:45,434 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:51:46,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:51:47,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:49,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:50,011 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:51:50,011 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:51:51,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:51:53,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:05,142 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:52:05,143 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:52:05,487 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:52:05,487 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:52:05,487 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:52:06,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:52:07,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:08,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:09,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:11,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:11,575 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:52:13,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:20,274 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:52:20,274 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:52:25,572 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:52:25,573 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:52:25,573 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:52:26,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:52:27,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:28,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:29,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:31,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:33,349 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:35,405 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:52:35,406 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:52:41,654 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:52:45,644 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:52:45,645 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:52:45,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:52:46,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:52:47,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:48,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:49,356 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:50,544 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:52:50,544 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:52:51,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:52:53,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:05,673 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:53:05,674 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:53:05,803 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:53:05,803 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:53:05,804 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:53:06,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:53:08,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:10,365 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:11,732 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:53:12,366 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:14,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:20,839 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:53:20,840 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:53:25,730 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:53:25,731 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:53:25,731 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:53:26,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:53:28,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:30,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:32,374 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:34,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:35,970 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:53:35,970 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:53:41,806 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:53:45,769 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:53:45,770 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:53:45,770 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:53:46,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:53:48,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:50,382 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:51,127 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:53:51,127 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:53:52,383 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:53:54,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:05,868 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:54:05,869 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:54:05,869 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:54:06,257 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:54:06,257 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:54:06,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:54:08,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:10,391 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:11,876 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:54:12,392 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:14,392 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:21,394 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:54:21,395 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:54:25,927 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:54:25,927 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:54:25,928 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:54:26,400 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:54:28,400 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:30,401 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:32,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:34,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:36,526 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:54:36,526 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:54:41,947 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:54:45,988 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:54:45,989 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:54:45,989 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:54:46,408 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:54:48,409 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:50,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:51,657 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:54:51,657 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:54:52,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:54:54,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:06,079 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:55:06,079 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:55:06,080 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:55:06,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:55:06,828 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:55:06,829 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:55:08,417 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:10,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:12,018 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:55:12,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:14,420 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:21,969 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:55:21,970 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:55:26,120 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:55:26,120 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:55:26,121 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:55:26,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:55:28,426 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:30,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:32,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:34,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:37,101 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:55:37,102 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:55:42,088 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:55:46,218 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:55:46,219 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:55:46,219 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:55:46,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:55:48,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:50,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:52,231 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:55:52,232 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:55:52,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:55:54,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:06,279 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:56:06,279 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:56:06,280 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:56:06,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:56:07,374 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:56:07,374 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:56:08,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:10,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:12,159 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:56:12,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:14,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:22,521 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:56:22,522 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:56:26,319 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:56:26,320 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:56:26,320 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:56:26,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:56:28,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:30,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:32,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:34,452 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:37,653 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:56:37,653 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:56:42,235 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:56:46,375 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:56:46,375 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:56:46,376 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:56:46,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:56:48,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:50,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:52,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:56:52,791 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:56:52,792 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:56:54,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:06,460 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:57:06,461 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:57:06,461 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:57:06,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:57:07,990 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:57:07,991 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:57:08,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:10,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:12,314 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:57:12,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:14,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:23,144 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:57:23,145 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:57:26,501 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:57:26,502 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:57:26,502 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:57:27,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:57:28,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:30,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:32,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:34,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:38,276 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:57:38,276 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:57:42,391 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:57:46,575 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:57:46,575 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:57:46,576 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:57:47,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:57:48,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:50,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:52,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:57:53,409 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:57:53,409 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:57:54,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:06,616 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:58:06,617 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:58:06,617 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:58:07,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:58:08,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:08,603 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:58:08,603 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:58:11,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:12,469 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:58:13,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:15,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:23,735 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:58:23,736 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:58:26,669 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:58:26,670 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:58:26,671 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:58:27,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:58:28,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:29,498 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:31,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:33,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:35,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:38,869 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:58:38,869 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:58:42,546 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:58:46,719 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:58:46,720 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:58:46,720 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:58:47,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:58:48,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:49,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:51,507 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:53,508 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:58:54,002 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:58:54,002 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:58:55,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:06,770 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:59:06,771 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:59:06,771 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:59:07,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:59:09,135 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:59:09,135 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:59:09,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:11,515 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:12,624 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:59:13,516 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:15,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:24,270 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:59:24,270 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:59:26,811 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:59:26,812 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:59:26,812 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:59:27,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:59:29,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:31,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:33,524 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:35,525 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:39,403 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:59:39,403 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:59:42,703 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 21:59:46,883 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 21:59:46,883 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 21:59:46,884 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 21:59:47,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 21:59:49,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:51,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:53,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 21:59:54,534 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 21:59:54,534 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 21:59:55,534 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:06,973 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:00:06,973 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:00:06,974 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:00:07,540 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:00:09,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:09,692 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:00:09,692 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:00:11,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:12,780 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:00:13,543 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:15,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:24,851 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:00:24,852 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:00:27,027 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:00:27,027 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:00:27,028 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:00:27,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:00:29,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:31,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:33,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:35,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:40,006 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:00:40,007 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:00:42,858 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:00:47,075 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:00:47,076 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:00:47,076 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:00:47,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:00:49,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:51,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:53,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:00:55,141 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:00:55,141 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:00:55,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:07,123 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:01:07,123 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:01:07,124 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:01:07,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:01:09,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:10,310 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:01:10,311 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:01:11,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:12,936 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:01:13,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:15,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:25,462 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:01:25,463 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:01:27,184 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:01:27,184 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:01:27,185 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:01:27,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:01:29,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:31,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:33,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:35,581 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:40,597 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:01:40,597 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:01:43,018 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:01:47,238 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:01:47,239 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:01:47,239 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:01:47,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:01:49,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:51,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:53,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:55,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:01:55,759 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:01:55,759 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:02:07,280 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:02:07,280 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:02:07,281 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:02:07,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:02:09,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:10,937 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:02:10,937 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:02:11,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:13,097 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:02:13,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:15,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:26,066 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:02:26,066 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:02:27,344 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:02:27,345 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:02:27,345 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:02:27,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:02:29,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:31,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:33,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:35,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:41,201 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:02:41,202 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:02:43,174 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:02:47,413 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:02:47,413 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:02:47,414 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:02:47,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:02:49,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:51,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:53,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:55,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:02:56,335 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:02:56,336 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:03:07,462 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:03:07,463 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:03:07,463 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:03:07,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:03:09,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:11,465 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:03:11,466 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:03:11,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:13,251 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:03:13,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:15,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:26,598 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:03:26,598 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:03:27,501 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:03:27,502 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:03:27,502 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:03:27,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:03:29,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:31,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:33,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:35,637 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:41,730 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:03:41,730 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:03:43,329 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:03:47,556 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:03:47,556 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:03:47,557 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:03:47,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:03:49,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:51,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:53,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:55,646 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:03:56,861 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:03:56,862 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:04:07,642 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:04:07,643 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:04:07,643 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:04:07,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:04:09,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:11,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:11,992 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:04:11,992 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:04:13,407 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:04:13,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:15,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:27,682 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:04:27,682 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:04:27,682 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:04:28,220 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:04:28,220 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:04:28,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:04:29,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:31,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:33,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:35,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:43,359 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:04:43,360 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:04:43,492 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:04:47,790 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:04:47,791 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:04:47,791 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:04:48,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:04:49,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:51,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:53,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:55,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:04:58,494 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:04:58,494 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:05:07,885 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:05:07,886 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:05:07,886 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:05:08,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:05:09,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:11,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:13,562 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:05:13,624 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:05:13,624 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:05:13,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:15,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:27,926 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:05:27,927 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:05:27,928 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:05:28,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:05:28,762 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:05:28,763 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:05:29,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:31,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:33,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:35,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:43,641 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:05:43,902 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:05:43,902 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:05:47,966 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:05:47,966 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:05:47,966 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:05:48,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:05:50,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:52,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:54,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:56,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:05:59,036 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:05:59,036 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:06:08,044 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:06:08,044 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:06:08,045 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:06:08,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:06:10,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:12,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:13,719 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:06:14,168 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:06:14,168 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:06:14,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:16,702 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:28,101 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:06:28,102 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:06:28,102 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:06:28,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:06:29,305 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:06:29,306 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:06:30,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:32,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:34,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:36,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:43,795 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:06:44,437 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:06:44,437 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:06:48,175 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:06:48,176 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:06:48,176 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:06:48,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:06:50,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:52,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:54,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:56,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:06:59,577 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:06:59,577 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:07:08,217 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:07:08,218 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:07:08,218 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:07:08,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:07:10,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:12,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:13,874 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:07:14,715 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:07:14,716 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:07:14,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:16,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:28,308 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:07:28,308 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:07:28,309 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:07:28,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:07:29,846 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:07:29,847 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:07:30,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:32,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:34,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:36,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:43,953 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:07:44,986 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:07:44,987 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:07:48,348 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:07:48,349 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:07:48,349 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:07:48,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:07:50,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:52,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:54,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:07:56,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:00,119 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:08:00,119 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:08:08,429 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:08:08,429 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:08:08,430 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:08:08,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:08:10,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:12,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:14,031 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:08:14,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:15,252 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:08:15,253 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:08:16,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:28,489 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:08:28,489 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:08:28,490 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:08:28,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:08:30,386 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:08:30,386 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:08:30,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:32,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:34,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:36,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:44,101 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:08:45,525 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:08:45,525 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:08:48,542 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:08:48,542 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:08:48,543 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:08:48,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:08:50,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:52,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:54,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:08:56,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:00,657 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:09:00,657 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:09:08,588 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:09:08,589 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:09:08,589 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:09:08,777 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:09:10,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:12,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:14,168 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:09:14,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:15,787 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:09:15,787 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:09:16,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:28,628 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:09:28,629 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:09:28,629 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:09:28,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:09:30,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:30,932 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:09:30,932 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:09:32,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:34,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:36,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:44,237 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:09:46,063 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:09:46,063 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:09:48,668 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:09:48,668 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:09:48,669 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:09:48,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:09:50,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:52,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:54,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:09:56,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:01,195 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:10:01,195 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:10:08,742 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:10:08,743 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:10:08,743 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:10:08,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:10:10,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:12,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:14,306 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:10:14,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:16,341 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:10:16,342 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:10:16,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:28,800 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:10:28,801 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:10:28,801 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:10:28,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:10:30,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:31,476 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:10:31,476 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:10:32,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:34,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:36,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:44,374 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:10:46,625 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:10:46,626 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:10:48,842 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:10:48,843 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:10:48,843 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:10:49,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:10:50,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:52,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:54,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:10:56,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:01,755 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:11:01,756 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:11:08,883 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:11:08,883 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:11:08,884 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:11:09,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:11:10,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:12,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:14,445 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:11:14,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:16,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:16,894 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:11:16,894 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:11:28,950 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:11:28,950 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:11:28,951 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:11:29,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:11:30,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:32,047 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:11:32,047 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:11:32,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:34,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:36,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:44,514 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:11:47,187 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:11:47,187 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:11:49,003 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:11:49,003 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:11:49,004 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:11:49,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:11:50,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:52,847 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:54,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:11:56,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:02,337 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:12:02,337 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:12:09,047 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:12:09,048 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:12:09,048 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:12:09,855 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:12:10,855 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:12,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:14,583 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:12:14,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:16,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:17,470 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:12:17,470 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:12:29,146 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:12:29,147 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:12:29,147 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:12:29,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:12:30,864 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:32,612 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:12:32,612 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:12:33,865 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:35,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:37,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:44,653 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:12:47,750 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:12:47,751 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:12:49,187 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:12:49,188 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:12:49,188 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:12:49,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:12:51,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:53,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:55,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:12:57,875 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:02,879 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:13:02,880 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:13:09,227 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:13:09,227 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:13:09,228 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:13:09,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:13:11,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:13,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:14,722 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:13:15,882 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:17,883 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:18,010 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:13:18,011 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:13:29,268 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:13:29,269 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:13:29,269 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:13:29,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:13:31,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:33,143 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:13:33,143 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:13:33,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:35,891 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:37,892 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:44,793 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:13:48,274 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:13:48,275 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:13:49,310 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:13:49,311 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:13:49,311 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:13:49,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:13:51,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:53,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:55,899 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:13:57,900 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:03,406 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:14:03,407 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:14:09,425 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:14:09,425 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:14:09,426 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:14:09,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:14:11,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:13,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:14,866 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:14:15,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:17,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:18,537 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:14:18,537 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:14:29,489 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:14:29,490 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:14:29,491 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:14:29,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:14:31,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:33,668 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:14:33,668 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:14:33,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:35,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:37,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:44,940 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:14:48,802 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:14:48,802 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:14:49,570 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:14:49,570 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:14:49,571 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:14:49,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:14:51,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:53,922 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:55,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:14:57,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:03,930 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:15:03,931 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:15:09,625 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:15:09,625 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:15:09,626 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:15:09,928 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:15:11,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:13,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:15,013 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:15:15,931 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:17,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:19,062 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:15:19,062 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:15:29,664 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:15:29,664 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:15:29,665 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:15:29,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:15:31,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:33,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:34,219 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:15:34,220 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:15:35,939 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:37,940 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:45,088 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:15:49,349 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:15:49,349 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:15:49,708 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:15:49,709 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:15:49,709 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:15:49,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:15:51,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:53,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:55,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:15:57,948 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:04,481 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:16:04,481 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:16:09,764 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:16:09,765 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:16:09,765 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:16:09,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:16:11,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:13,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:15,166 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:16:15,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:17,956 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:19,611 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:16:19,611 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:16:29,803 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:16:29,804 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:16:29,804 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:16:29,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:16:31,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:33,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:34,743 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:16:34,743 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:16:35,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:37,965 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:45,240 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:16:49,861 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:16:49,862 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:16:49,862 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:16:49,901 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:16:49,901 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:16:49,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:16:51,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:53,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:55,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:16:57,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:05,084 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:17:05,085 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:17:09,900 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:17:09,901 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:17:09,901 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:17:09,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:17:11,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:13,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:15,318 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:17:15,980 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:17,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:20,213 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:17:20,213 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:17:29,940 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:17:29,940 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:17:29,941 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:17:29,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:17:31,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:33,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:35,346 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:17:35,347 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:17:35,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:37,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:45,411 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:17:50,042 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:17:50,042 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:17:50,043 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:17:50,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:17:50,486 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:17:50,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:17:51,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:53,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:55,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:17:57,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:05,626 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:18:05,626 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:18:10,116 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:18:10,116 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:18:10,117 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:18:11,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:18:12,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:14,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:15,488 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:18:16,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:18,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:20,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:20,759 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:18:20,760 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:18:22,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:24,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:27,009 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:29,010 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:31,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:33,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:35,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:35,895 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:18:35,895 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:18:37,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:39,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:41,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:43,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:45,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:45,562 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:18:47,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:49,019 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:51,020 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:51,025 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:18:51,025 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:18:53,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:55,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:57,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:18:59,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:01,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:03,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:05,027 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:06,188 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:19:06,188 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:19:07,028 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:09,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:11,030 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:13,987 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 22:19:13,988 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 22:19:13,988 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:19:14,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 22:19:15,639 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 22:19:21,321 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:19:21,321 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:19:24,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 22:19:26,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log