diff --git "a/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20210716_223350-8eukt20m/logs/debug-internal.log" @@ -0,0 +1,3242 @@ +2021-07-16 22:33:51,421 INFO MainThread:799749 [internal.py:wandb_internal():88] W&B internal server running at pid: 799749, started at: 2021-07-16 22:33:51.421464 +2021-07-16 22:33:51,423 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: check_version +2021-07-16 22:33:51,423 INFO WriterThread:799749 [datastore.py:open_for_write():80] open: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/run-8eukt20m.wandb +2021-07-16 22:33:51,424 DEBUG SenderThread:799749 [sender.py:send():179] send: header +2021-07-16 22:33:51,424 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: check_version +2021-07-16 22:33:51,461 DEBUG SenderThread:799749 [sender.py:send():179] send: run +2021-07-16 22:33:51,635 INFO SenderThread:799749 [dir_watcher.py:__init__():168] watching files in: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files +2021-07-16 22:33:51,635 INFO SenderThread:799749 [sender.py:_start_run_threads():716] run started: 8eukt20m with start time 1626474830 +2021-07-16 22:33:51,635 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:33:51,636 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: run_start +2021-07-16 22:33:51,636 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:33:52,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:33:52,760 DEBUG HandlerThread:799749 [meta.py:__init__():39] meta init +2021-07-16 22:33:52,760 DEBUG HandlerThread:799749 [meta.py:__init__():53] meta init done +2021-07-16 22:33:52,760 DEBUG HandlerThread:799749 [meta.py:probe():210] probe +2021-07-16 22:33:52,762 DEBUG HandlerThread:799749 [meta.py:_setup_git():200] setup git +2021-07-16 22:33:52,791 DEBUG HandlerThread:799749 [meta.py:_setup_git():207] setup git done +2021-07-16 22:33:52,791 DEBUG HandlerThread:799749 [meta.py:_save_pip():57] save pip +2021-07-16 22:33:52,791 DEBUG HandlerThread:799749 [meta.py:_save_pip():71] save pip done +2021-07-16 22:33:52,791 DEBUG HandlerThread:799749 [meta.py:probe():252] probe done +2021-07-16 22:33:52,794 DEBUG SenderThread:799749 [sender.py:send():179] send: files +2021-07-16 22:33:52,795 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now +2021-07-16 22:33:52,802 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:33:52,804 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:33:52,932 DEBUG SenderThread:799749 [sender.py:send():179] send: config +2021-07-16 22:33:52,933 DEBUG SenderThread:799749 [sender.py:send():179] send: config +2021-07-16 22:33:52,933 DEBUG SenderThread:799749 [sender.py:send():179] send: config +2021-07-16 22:33:53,274 INFO Thread-11 :799749 [upload_job.py:push():137] Uploaded file /tmp/tmph3_p5zqlwandb/1dulgurl-wandb-metadata.json +2021-07-16 22:33:53,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-metadata.json +2021-07-16 22:33:53,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:33:53,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_created():216] file/dir created: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/requirements.txt +2021-07-16 22:34:07,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:34:07,944 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:34:09,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:34:11,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:34:20,845 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:34:22,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/config.yaml +2021-07-16 22:34:23,093 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:34:23,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:34:38,223 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:34:38,223 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:34:50,918 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:34:53,364 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:34:53,364 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:35:08,496 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:35:08,497 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:35:20,988 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:35:23,633 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:35:23,633 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:35:38,766 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:35:38,767 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:35:51,061 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:35:53,898 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:35:53,899 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:36:09,029 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:36:09,030 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:36:21,136 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:36:24,164 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:36:24,165 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:36:39,297 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:36:39,297 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:36:51,209 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:36:54,434 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:36:54,434 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:37:09,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:37:09,567 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:37:21,282 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:37:24,702 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:37:24,702 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:37:39,836 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:37:39,836 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:37:51,348 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:37:55,000 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:37:55,001 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:38:10,188 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:38:10,189 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:38:21,405 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:38:25,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:38:25,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:38:40,510 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:38:40,510 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:38:51,470 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:38:55,642 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:38:55,642 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:39:10,782 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:39:10,783 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:39:21,529 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:39:25,916 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:39:25,916 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:39:41,047 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:39:41,047 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:39:43,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:39:51,590 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:39:56,193 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:39:56,194 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:40:11,336 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:40:11,336 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:40:21,663 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:40:26,468 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:40:26,469 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:40:41,598 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:40:41,599 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:40:51,742 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:40:56,730 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:40:56,731 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:41:11,864 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:41:11,864 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:41:21,819 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:41:27,001 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:41:27,002 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:41:42,135 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:41:42,135 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:41:49,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:41:51,894 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:41:57,281 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:41:57,282 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:42:12,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:42:12,432 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:42:21,969 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:42:27,567 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:42:27,568 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:42:42,699 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:42:42,699 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:42:52,047 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:42:57,830 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:42:57,831 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:43:12,963 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:43:12,964 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:43:22,126 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:43:28,106 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:43:28,107 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:43:43,239 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:43:43,239 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:43:52,228 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:43:58,369 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:43:58,369 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:44:01,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:04,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:06,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:08,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:10,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:12,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:13,579 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:44:13,580 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:44:14,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:16,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:20,459 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:44:20,460 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:44:20,460 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:44:20,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:44:21,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:22,301 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:44:22,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:24,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:26,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:28,849 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:44:28,850 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:44:28,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:30,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:32,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:34,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:40,688 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:44:40,688 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:44:40,689 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:44:40,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:40,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:44:41,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:43,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:44,084 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:44:44,084 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:44:45,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:47,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:49,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:51,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:52,377 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:44:53,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:55,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:44:59,278 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:44:59,278 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:45:00,892 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:45:00,892 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:45:00,893 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:45:00,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:45:01,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:02,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:03,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:05,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:07,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:09,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:12,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:14,511 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:45:14,511 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:45:14,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:16,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:21,069 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:45:21,070 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:45:21,070 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:45:21,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:45:22,452 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:45:22,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:24,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:26,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:28,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:29,751 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:45:29,752 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:45:30,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:32,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:34,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:36,932 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:41,275 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:45:41,276 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:45:41,276 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:45:41,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:45:42,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:44,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:44,974 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:45:44,975 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:45:46,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:48,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:50,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:52,524 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:45:53,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:55,941 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:45:57,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:00,198 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:46:00,199 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:46:01,495 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:46:01,496 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:46:01,496 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:46:01,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:46:02,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:03,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:05,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:07,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:09,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:11,950 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:13,951 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:15,337 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:46:15,337 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:46:15,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:17,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:21,698 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:46:21,699 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:46:21,699 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:46:21,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:46:22,599 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:46:23,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:25,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:27,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:29,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:30,528 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:46:30,528 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:46:31,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:34,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:36,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:41,971 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:46:41,972 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:46:41,972 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:46:42,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:46:43,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:44,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:45,767 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:46:45,767 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:46:46,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:48,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:50,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:52,679 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:46:52,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:54,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:56,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:46:58,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:00,937 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:47:00,937 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:47:02,133 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:47:02,134 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:47:02,134 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:47:02,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:47:03,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:04,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:06,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:08,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:10,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:12,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:14,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:16,085 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:47:16,085 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:47:16,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:22,335 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:47:22,335 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:47:22,336 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:47:22,749 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:47:22,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:22,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:47:23,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:24,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:27,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:29,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:31,364 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:47:31,364 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:47:31,990 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:33,991 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:35,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:37,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:42,654 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:47:42,654 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:47:42,655 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:47:42,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:47:43,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:45,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:46,521 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:47:46,521 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:47:47,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:49,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:52,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:52,822 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:47:54,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:56,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:47:58,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:01,699 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:48:01,699 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:48:02,900 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:48:02,901 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:48:02,901 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:48:03,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:48:04,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:05,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:07,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:09,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:11,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:13,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:15,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:16,835 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:48:16,835 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:48:17,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:19,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:22,897 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:48:23,092 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:48:23,092 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:48:23,093 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:48:24,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:48:25,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:27,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:29,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:31,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:32,092 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:48:32,093 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:48:33,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:35,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:37,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:39,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:43,343 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:48:43,343 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:48:43,344 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:48:44,025 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:48:45,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:47,370 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:48:47,371 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:48:48,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:50,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:52,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:52,971 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:48:54,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:56,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:48:58,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:02,527 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:49:02,528 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:49:03,607 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:49:03,608 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:49:03,608 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:49:04,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:49:05,035 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:06,036 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:08,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:10,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:12,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:14,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:16,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:17,659 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:49:17,659 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:49:18,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:23,046 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:49:23,871 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:49:23,872 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:49:23,872 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:49:24,044 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:49:26,045 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:28,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:30,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:32,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:32,802 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:49:32,802 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:49:35,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:37,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:39,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:44,094 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:49:44,095 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:49:44,095 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:49:45,054 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:49:46,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:47,055 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:48,079 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:49:48,079 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:49:49,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:51,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:53,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:53,120 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:49:55,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:57,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:49:59,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:03,364 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:50:03,364 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:50:04,296 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:50:04,296 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:50:04,297 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:50:05,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:50:06,065 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:07,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:09,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:11,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:13,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:15,070 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:18,071 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:18,501 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:50:18,502 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:50:20,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:23,197 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:50:24,609 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:50:24,609 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:50:24,610 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:50:25,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:50:26,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:28,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:30,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:32,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:33,662 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:50:33,662 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:50:34,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:36,080 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:38,081 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:40,082 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:44,850 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:50:44,850 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:50:44,851 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:50:45,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:50:46,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:48,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:48,985 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:50:48,985 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:50:50,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:52,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:53,274 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:50:54,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:57,090 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:50:59,091 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:01,092 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:04,332 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:51:04,332 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:51:05,154 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:51:05,155 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:51:05,155 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:51:06,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:51:07,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:09,095 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:11,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:13,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:15,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:17,099 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:19,100 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:19,463 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:51:19,463 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:51:23,348 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:51:25,374 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:51:25,374 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:51:25,375 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:51:26,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:51:27,103 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:29,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:31,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:33,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:34,630 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:51:34,631 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:51:35,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:37,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:39,109 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:45,600 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:51:45,600 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:51:45,601 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:51:46,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:51:47,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:48,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:49,981 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:51:49,981 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:51:50,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:52,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:53,421 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:51:54,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:56,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:51:58,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:00,118 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:05,212 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:52:05,212 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:52:05,889 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:52:05,889 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:52:05,889 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:52:06,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:52:07,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:08,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:10,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:12,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:14,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:16,126 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:18,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:20,127 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:20,344 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:52:20,345 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:52:23,493 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:52:26,124 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:52:26,124 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:52:26,125 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:52:26,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:52:27,130 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:28,131 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:31,132 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:33,133 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:35,134 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:35,522 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:52:35,523 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:52:37,135 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:39,136 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:41,137 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:46,289 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:52:46,290 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:52:46,290 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:52:47,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:47,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:52:48,140 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:49,141 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:50,696 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:52:50,697 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:52:51,142 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:53,143 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:53,565 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:52:55,144 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:57,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:52:59,145 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:01,146 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:05,909 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:53:05,910 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:53:06,474 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:53:06,474 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:53:06,475 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:53:07,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:53:08,149 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:10,150 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:12,151 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:14,152 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:16,153 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:18,154 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:20,155 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:21,041 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:53:21,042 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:53:23,641 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:53:26,736 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:53:26,737 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:53:26,737 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:53:27,157 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:53:28,158 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:30,159 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:32,160 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:34,161 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:36,162 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:36,178 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:53:36,178 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:53:39,163 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:41,164 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:43,165 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:46,946 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:53:46,947 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:53:46,947 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:53:47,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:53:48,167 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:49,168 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:51,169 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:51,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:53:51,432 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:53:53,170 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:53,716 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:53:55,171 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:57,172 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:53:59,173 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:01,174 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:03,175 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:06,571 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:54:06,571 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:54:07,188 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:54:07,188 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:54:07,189 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:54:08,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:54:09,177 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:11,178 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:13,179 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:15,180 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:17,181 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:19,182 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:21,183 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:21,703 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:54:21,703 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:54:23,793 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:54:27,449 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:54:27,449 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:54:27,449 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:54:28,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:54:29,186 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:30,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:32,187 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:34,188 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:36,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:37,005 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:54:37,006 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:54:38,190 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:40,191 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:42,192 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:47,704 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:54:47,704 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:54:47,705 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:54:48,195 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:54:49,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:50,196 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:52,180 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:54:52,180 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:54:52,197 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:53,869 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:54:54,198 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:56,199 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:54:58,200 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:00,201 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:02,202 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:07,324 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:55:07,325 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:55:07,894 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:55:07,894 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:55:07,895 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:55:08,205 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:55:09,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:10,206 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:12,207 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:15,209 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:17,210 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:19,211 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:21,212 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:22,455 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:55:22,455 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:55:23,213 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:23,944 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:55:28,035 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:55:28,035 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:55:28,036 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:55:28,215 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:55:29,216 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:31,217 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:33,218 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:35,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:37,219 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:37,624 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:55:37,625 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:55:39,220 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:41,221 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:43,222 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:48,252 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:55:48,253 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:55:48,253 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:55:49,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:49,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:55:50,225 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:51,226 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:52,892 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:55:52,892 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:55:53,227 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:54,020 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:55:56,228 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:55:58,229 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:00,230 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:02,231 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:08,023 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:56:08,023 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:56:08,437 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:56:08,437 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:56:08,438 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:56:09,234 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:56:10,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:12,235 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:14,236 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:16,237 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:18,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:20,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:22,240 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:23,152 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:56:23,153 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:56:24,095 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:56:28,603 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:56:28,604 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:56:28,604 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:56:29,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:56:30,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:32,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:34,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:36,246 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:38,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:38,306 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:56:38,307 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:56:40,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:42,250 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:48,824 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:56:48,825 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:56:48,825 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:56:49,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:56:50,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:51,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:53,255 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:53,504 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:56:53,505 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:56:54,170 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:56:55,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:57,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:56:59,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:01,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:03,260 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:08,668 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:57:08,668 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:57:09,081 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:57:09,082 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:57:09,082 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:57:09,262 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:57:11,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:13,264 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:15,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:17,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:19,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:21,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:23,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:23,849 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:57:23,850 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:57:24,245 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:57:29,267 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:57:29,267 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:57:29,268 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:57:29,271 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:57:31,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:33,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:36,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:38,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:39,105 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:57:39,105 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:57:40,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:42,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:49,452 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:57:49,453 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:57:49,453 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:57:50,281 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:57:51,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:52,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:54,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:54,320 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:57:54,357 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:57:54,357 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:57:56,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:57:58,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:00,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:02,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:04,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:09,548 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:58:09,548 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:58:09,684 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:58:09,685 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:58:09,686 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:58:10,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:58:11,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:12,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:14,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:16,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:18,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:20,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:22,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:24,396 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:58:24,753 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:58:24,754 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:58:29,810 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:58:29,810 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:58:29,811 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:58:30,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:58:31,300 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:33,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:35,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:37,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:39,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:39,904 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:58:39,904 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:58:41,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:43,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:45,307 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:50,113 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:58:50,113 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:58:50,113 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:58:50,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:58:51,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:52,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:53,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:54,472 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:58:55,142 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:58:55,142 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:58:55,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:58:58,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:00,314 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:02,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:04,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:10,283 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:59:10,283 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:59:10,427 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:59:10,427 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:59:10,428 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:59:11,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:59:12,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:14,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:16,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:18,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:20,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:22,325 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:24,326 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:24,546 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:59:25,413 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:59:25,414 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:59:30,733 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:59:30,733 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:59:30,734 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:59:31,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:59:32,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:34,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:36,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:38,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:40,584 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:59:40,584 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:59:41,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:43,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:45,336 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:51,031 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 22:59:51,031 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 22:59:51,032 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 22:59:51,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 22:59:52,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:53,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:54,623 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 22:59:55,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:55,838 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 22:59:55,839 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 22:59:57,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 22:59:59,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:01,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:03,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:05,345 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:10,973 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:00:10,974 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:00:11,372 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:00:11,372 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:00:11,372 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:00:12,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:00:13,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:15,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:17,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:19,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:21,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:23,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:24,697 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:00:25,355 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:26,103 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:00:26,103 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:00:31,884 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:00:31,885 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:00:31,885 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:00:32,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:00:33,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:35,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:37,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:39,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:41,233 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:00:41,234 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:00:42,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:44,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:46,364 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:52,161 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:00:52,162 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:00:52,162 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:00:52,367 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:00:53,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:54,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:54,773 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:00:56,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:00:56,456 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:00:56,457 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:00:58,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:00,371 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:02,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:04,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:06,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:11,704 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:01:11,704 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:01:12,295 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:01:12,295 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:01:12,296 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:01:12,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:01:13,377 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:14,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:16,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:18,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:20,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:22,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:24,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:24,844 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:01:26,835 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:01:26,836 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:01:32,508 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:01:32,508 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:01:32,508 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:01:33,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:33,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:01:34,387 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:36,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:39,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:41,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:42,112 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:01:42,113 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:01:43,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:45,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:47,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:52,733 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:01:52,733 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:01:52,734 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:01:53,396 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:01:54,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:54,917 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:01:55,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:57,385 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:01:57,385 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:01:57,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:01:59,399 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:01,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:03,401 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:05,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:07,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:12,582 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:02:12,583 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:02:12,919 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:02:12,919 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:02:12,920 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:02:13,406 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:02:14,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:15,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:17,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:19,409 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:21,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:23,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:24,991 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:02:25,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:27,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:27,720 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:02:27,720 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:02:33,115 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:02:33,116 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:02:33,116 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:02:33,415 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:02:34,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:35,416 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:37,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:39,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:41,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:42,924 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:02:42,924 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:02:43,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:45,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:53,276 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:02:53,276 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:02:53,277 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:02:53,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:02:54,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:55,067 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:02:55,425 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:57,426 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:02:58,117 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:02:58,118 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:03:00,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:02,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:04,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:06,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:13,288 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:03:13,288 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:03:13,605 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:03:13,605 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:03:13,605 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:03:14,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:14,434 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:03:16,435 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:18,436 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:20,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:22,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:24,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:25,143 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:03:26,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:28,417 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:03:28,417 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:03:28,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:33,782 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:03:33,782 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:03:33,782 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:03:34,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:03:35,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:36,444 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:38,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:40,445 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:42,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:43,552 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:03:43,552 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:03:44,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:46,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:53,950 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:03:53,950 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:03:53,950 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:03:54,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:03:55,218 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:03:55,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:56,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:58,453 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:03:58,820 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:03:58,820 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:04:01,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:03,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:05,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:07,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:09,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:13,964 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:04:13,965 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:04:14,212 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:04:14,213 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:04:14,213 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:04:14,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:04:15,461 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:17,462 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:19,463 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:21,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:23,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:25,292 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:04:25,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:27,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:29,095 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:04:29,096 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:04:29,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:34,437 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:04:34,437 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:04:34,438 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:04:34,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:04:35,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:37,471 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:39,472 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:41,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:43,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:44,318 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:04:44,318 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:04:45,474 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:47,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:50,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:54,710 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:04:54,710 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:04:54,711 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:04:55,363 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:04:55,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:55,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:04:56,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:58,480 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:04:59,602 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:04:59,603 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:05:00,481 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:02,482 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:04,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:06,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:08,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:14,770 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:05:14,770 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:05:14,920 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:05:14,920 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:05:14,921 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:05:15,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:05:16,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:18,489 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:20,490 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:22,491 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:24,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:25,430 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:05:26,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:28,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:29,904 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:05:29,904 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:05:35,157 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:05:35,158 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:05:35,158 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:05:35,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:05:36,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:38,498 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:40,499 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:42,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:44,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:45,105 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:05:45,105 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:05:47,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:49,503 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:55,357 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:05:55,357 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:05:55,358 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:05:55,499 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:05:55,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:05:56,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:57,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:05:59,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:00,243 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:06:00,244 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:06:01,508 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:03,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:05,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:07,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:09,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:15,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:06:15,422 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:06:15,592 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:06:15,592 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:06:15,593 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:06:16,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:16,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:06:17,516 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:19,517 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:21,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:23,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:25,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:25,573 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:06:27,521 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:29,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:30,553 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:06:30,553 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:06:35,861 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:06:35,862 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:06:35,862 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:06:36,525 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:06:38,526 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:40,527 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:42,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:44,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:45,802 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:06:45,802 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:06:46,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:48,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:50,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:06:55,647 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:06:56,114 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:06:56,114 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:06:56,115 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:06:56,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:06:58,535 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:00,536 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:01,028 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:07:01,028 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:07:02,537 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:04,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:06,538 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:08,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:10,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:16,161 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:07:16,162 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:07:16,450 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:07:16,450 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:07:16,451 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:07:16,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:07:18,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:20,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:22,545 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:24,546 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:25,720 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:07:27,547 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:29,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:31,291 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:07:31,291 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:07:31,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:36,723 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:07:36,723 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:07:36,723 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:07:37,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:07:39,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:41,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:43,555 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:45,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:46,497 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:07:46,498 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:07:47,556 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:49,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:07:55,792 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:07:56,912 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:07:56,912 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:07:56,913 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:07:57,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:07:59,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:01,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:01,839 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:08:01,839 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:08:03,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:05,564 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:07,565 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:09,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:11,566 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:17,038 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:08:17,039 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:08:17,170 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:08:17,171 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:08:17,171 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:08:17,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:17,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:08:19,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:20,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:22,571 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:24,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:25,866 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:08:26,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:28,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:30,575 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:32,255 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:08:32,255 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:08:37,276 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:08:37,277 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:08:37,277 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:08:37,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:37,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:08:38,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:40,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:42,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:44,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:46,582 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:47,432 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:08:47,432 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:08:49,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:51,585 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:08:55,936 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:08:57,630 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:08:57,630 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:08:57,631 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:08:58,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:08:59,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:00,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:01,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:02,631 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:09:02,631 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:09:03,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:05,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:07,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:09,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:11,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:17,765 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:09:17,766 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:09:17,766 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:09:17,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:09:17,945 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:09:18,597 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:09:19,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:20,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:21,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:23,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:25,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:26,010 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:09:28,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:30,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:32,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:33,136 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:09:33,136 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:09:37,954 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:09:37,954 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:09:37,955 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:09:38,606 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:09:40,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:42,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:44,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:46,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:48,372 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:09:48,372 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:09:48,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:50,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:52,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:09:56,081 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:09:58,183 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:09:58,184 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:09:58,184 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:09:58,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:10:00,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:02,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:03,567 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:10:03,568 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:10:04,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:06,620 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:08,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:10,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:18,411 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:10:18,411 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:10:18,412 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:10:18,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:18,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:10:18,799 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:10:18,800 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:10:21,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:23,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:25,628 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:26,155 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:10:27,629 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:29,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:31,631 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:34,021 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:10:34,022 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:10:38,556 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:10:38,556 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:10:38,557 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:10:38,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:10:39,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:41,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:43,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:45,638 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:47,640 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:49,151 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:10:49,151 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:10:49,641 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:51,642 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:56,230 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:10:58,865 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:10:58,866 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:10:58,866 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:10:59,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:10:59,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:11:01,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:03,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:04,383 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:11:04,384 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:11:06,649 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:08,651 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:10,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:12,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:19,065 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:11:19,066 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:11:19,066 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:11:19,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:19,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:11:19,660 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:11:19,660 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:11:20,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:22,657 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:24,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:26,305 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:11:26,659 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:28,660 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:30,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:32,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:34,974 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:11:34,974 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:11:39,238 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:11:39,239 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:11:39,239 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:11:39,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:11:40,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:41,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:42,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:44,668 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:46,669 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:49,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:50,187 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:11:50,187 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:11:51,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:53,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:11:56,380 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:11:59,398 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:11:59,398 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:11:59,399 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:11:59,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:12:01,677 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:02,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:03,678 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:05,420 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:12:05,420 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:12:05,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:07,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:09,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:11,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:13,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:19,571 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:12:19,572 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:12:19,572 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:12:19,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:12:20,633 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:12:20,634 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:12:20,687 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:21,688 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:23,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:25,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:26,453 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:12:27,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:29,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:31,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:33,695 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:35,795 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:12:35,796 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:12:39,809 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:12:39,809 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:12:39,810 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:12:40,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:12:41,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:42,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:43,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:46,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:48,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:50,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:51,013 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:12:51,013 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:12:52,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:54,706 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:12:56,524 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:13:00,080 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:13:00,080 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:13:00,081 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:13:00,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:00,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:13:02,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:04,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:06,698 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:13:06,699 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:13:06,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:08,714 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:10,715 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:12,716 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:20,285 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:13:20,286 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:13:20,286 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:13:20,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:13:21,926 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:13:21,927 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:13:22,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:24,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:26,598 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:13:26,722 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:28,724 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:30,725 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:32,726 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:37,116 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:13:37,116 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:13:40,610 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:13:40,611 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:13:40,611 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:13:40,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:13:42,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:43,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:45,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:47,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:49,734 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:51,735 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:52,267 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:13:52,267 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:13:53,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:55,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:13:56,670 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:14:00,754 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:14:00,755 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:14:00,755 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:14:01,740 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:01,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:14:03,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:05,743 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:07,494 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:14:07,494 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:14:07,744 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:09,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:11,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:13,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:20,871 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:14:20,871 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:14:20,872 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:14:21,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:21,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:14:22,684 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:14:22,684 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:14:23,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:25,754 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:26,748 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:14:28,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:30,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:32,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:34,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:37,873 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:14:37,873 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:14:41,076 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:14:41,076 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:14:41,077 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:14:41,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:14:42,762 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:43,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:44,763 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:46,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:48,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:50,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:52,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:53,035 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:14:53,036 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:14:54,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:14:56,824 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:15:01,277 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:15:01,277 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:15:01,278 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:15:01,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:01,773 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:15:02,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:04,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:06,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:08,253 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:15:08,253 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:15:08,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:10,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:12,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:15,781 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:21,530 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:15:21,531 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:15:21,531 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:15:21,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:15:22,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:23,429 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:15:23,429 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:15:23,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:25,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:26,897 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:15:27,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:29,789 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:31,790 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:33,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:35,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:38,564 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:15:38,564 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:15:41,631 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:15:41,631 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:15:41,632 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:15:41,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:15:42,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:43,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:45,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:47,798 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:49,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:51,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:53,758 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:15:53,758 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:15:53,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:55,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:15:56,971 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:16:01,857 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:16:01,857 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:16:01,857 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:16:02,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:02,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:16:03,807 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:05,808 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:08,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:09,033 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:16:09,034 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:16:10,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:12,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:14,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:22,028 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:16:22,028 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:16:22,029 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:16:22,816 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:22,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:16:24,363 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:16:24,364 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:16:24,817 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:26,818 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:27,045 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:16:28,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:30,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:32,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:34,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:39,577 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:16:39,577 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:16:42,202 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:16:42,203 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:16:42,203 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:16:42,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:42,825 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:16:44,826 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:46,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:48,828 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:50,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:52,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:54,708 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:16:54,708 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:16:54,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:16:57,118 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:17:02,350 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:17:02,350 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:17:02,351 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:17:02,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:17:03,834 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:05,835 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:07,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:09,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:09,888 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:17:09,888 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:17:11,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:13,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:15,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:22,500 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:17:22,500 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:17:22,501 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:17:22,843 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:17:23,844 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:25,176 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:17:25,176 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:17:25,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:27,192 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:17:27,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:29,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:31,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:33,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:35,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:40,393 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:17:40,393 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:17:42,547 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:17:42,548 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:17:42,548 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:17:42,853 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:17:43,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:45,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:48,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:50,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:52,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:54,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:55,523 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:17:55,523 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:17:56,860 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:17:57,263 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:18:02,713 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:18:02,713 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:18:02,714 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:18:02,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:18:03,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:04,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:06,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:08,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:10,723 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:18:10,723 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:18:10,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:12,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:14,869 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:22,982 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:18:22,983 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:18:22,983 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:18:23,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:23,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:18:24,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:25,992 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:18:25,992 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:18:26,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:27,336 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:18:28,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:31,877 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:33,878 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:35,879 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:41,148 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:18:41,149 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:18:43,252 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:18:43,252 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:18:43,253 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:18:43,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:18:44,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:45,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:47,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:49,886 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:51,887 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:53,888 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:55,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:18:56,279 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:18:56,280 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:18:57,413 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:18:57,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:03,415 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:19:03,416 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:19:03,416 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:19:03,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:19:04,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:05,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:07,895 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:09,896 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:11,413 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:19:11,413 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:19:11,897 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:13,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:15,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:17,900 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:23,708 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:19:23,708 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:19:23,709 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:19:23,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:19:24,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:26,700 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:19:26,700 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:19:26,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:27,488 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:19:28,905 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:30,906 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:32,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:34,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:36,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:41,864 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:19:41,865 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:19:43,903 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:19:43,903 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:19:43,904 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:19:43,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:19:44,913 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:46,914 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:48,915 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:50,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:52,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:54,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:56,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:19:56,995 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:19:56,996 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:19:57,563 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:20:04,080 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:20:04,081 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:20:04,081 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:20:04,923 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:04,924 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:20:07,925 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:09,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:11,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:12,168 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:20:12,168 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:20:13,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:15,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:17,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:24,290 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:20:24,291 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:20:24,291 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:20:24,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:24,933 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:20:25,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:27,422 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:20:27,423 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:20:27,640 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:20:27,934 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:29,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:31,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:33,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:35,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:37,940 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:42,607 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:20:42,607 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:20:44,524 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:20:44,525 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:20:44,525 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:20:44,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:20:45,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:47,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:50,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:52,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:54,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:56,948 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:20:57,713 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:20:57,737 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:20:57,738 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:20:58,949 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:04,713 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:21:04,713 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:21:04,714 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:21:04,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:21:05,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:06,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:08,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:10,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:12,887 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:21:12,887 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:21:12,957 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:14,958 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:16,959 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:18,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:24,982 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:21:24,982 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:21:24,982 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:21:25,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:25,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:21:26,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:27,782 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:21:28,097 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:21:28,097 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:21:28,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:30,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:32,967 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:34,968 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:36,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:43,356 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:21:43,356 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:21:45,130 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:21:45,131 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:21:45,131 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:21:45,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:45,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:21:46,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:48,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:50,975 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:52,976 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:55,977 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:57,854 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:21:57,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:21:58,486 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:21:58,487 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:21:59,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:05,278 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:22:05,279 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:22:05,279 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:22:05,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:05,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:22:07,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:09,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:11,985 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:13,671 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:22:13,672 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:22:13,986 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:15,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:17,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:25,499 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:22:25,499 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:22:25,499 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:22:25,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:22:26,992 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:27,925 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:22:27,993 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:28,930 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:22:28,931 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:22:29,994 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:31,995 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:33,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:35,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:37,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:44,062 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:22:44,062 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:22:45,712 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:22:45,712 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:22:45,713 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:22:46,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:22:48,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:49,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:50,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:52,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:55,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:57,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:58,003 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:22:59,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:22:59,195 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:22:59,195 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:23:05,929 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:23:05,929 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:23:05,929 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:23:06,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:23:07,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:09,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:11,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:13,013 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:14,335 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:23:14,335 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:23:15,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:17,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:19,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:26,246 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:23:26,246 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:23:26,246 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:23:27,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:23:28,079 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:23:29,020 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:29,510 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:23:29,510 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:23:31,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:33,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:36,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:38,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:40,025 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:44,686 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:23:44,686 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:23:46,472 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:23:46,472 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:23:46,473 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:23:47,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:47,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:23:48,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:50,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:52,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:54,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:56,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:58,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:23:58,155 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:23:59,814 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:23:59,815 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:24:00,034 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:06,674 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:24:06,674 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:24:06,675 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:24:07,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:24:08,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:10,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:12,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:14,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:14,999 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:24:14,999 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:24:16,236 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:18,238 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:20,239 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:26,872 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:24:26,872 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:24:26,872 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:24:27,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:24:28,242 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:28,368 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:24:29,243 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:30,484 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:24:30,485 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:24:31,244 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:33,245 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:35,246 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:37,247 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:39,248 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:45,659 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:24:45,659 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:24:47,183 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:24:47,184 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:24:47,184 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:24:47,252 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:24:49,253 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:50,254 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:51,255 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:53,256 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:55,257 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:57,258 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:24:58,445 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:24:59,259 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:00,792 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:25:00,792 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:25:07,427 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:25:07,427 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:25:07,428 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:25:08,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:08,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:25:09,263 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:11,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:13,265 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:15,266 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:15,932 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:25:15,932 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:25:17,267 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:19,268 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:21,269 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:27,536 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:25:27,537 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:25:27,537 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:25:28,272 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:28,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:25:28,521 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:25:29,273 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:31,166 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:25:31,166 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:25:31,274 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:34,275 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:36,276 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:38,277 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:40,278 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:46,384 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:25:46,384 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:25:47,702 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:25:47,703 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:25:47,703 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:25:48,282 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:25:50,283 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:52,284 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:54,285 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:56,286 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:58,287 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:25:58,596 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:26:00,288 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:01,513 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:26:01,513 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:26:07,851 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:26:07,851 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:26:07,852 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:26:08,291 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:26:09,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:10,292 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:12,293 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:14,294 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:16,295 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:16,756 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:26:16,757 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:26:18,296 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:21,297 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:28,113 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:26:28,114 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:26:28,114 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:26:28,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:26:28,666 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:26:29,301 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:31,302 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:32,120 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:26:32,120 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:26:33,303 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:35,304 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:37,305 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:39,306 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:47,282 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:26:47,282 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:26:48,351 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:26:48,352 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:26:48,352 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:26:49,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:49,310 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:26:51,311 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:53,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:55,312 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:57,313 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:26:58,737 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:27:00,315 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:02,316 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:02,413 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:27:02,413 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:27:08,600 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:27:08,601 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:27:08,602 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:27:09,319 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:09,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:27:10,320 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:12,321 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:14,322 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:16,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:17,600 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:27:17,601 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:27:18,323 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:20,324 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:28,737 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:27:28,738 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:27:28,740 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:27:28,806 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:27:29,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:27:30,329 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:32,330 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:32,792 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:27:32,793 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:27:34,331 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:36,332 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:38,333 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:40,334 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:42,335 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:48,002 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:27:48,003 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:27:49,030 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:27:49,031 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:27:49,031 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:27:49,338 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:27:50,339 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:52,340 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:54,341 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:56,342 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:58,343 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:27:58,876 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:28:00,344 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:03,133 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:28:03,133 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:28:09,162 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:28:09,163 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:28:09,163 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:28:09,348 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:28:10,349 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:13,350 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:15,351 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:17,352 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:18,278 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:28:18,278 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:28:19,353 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:21,354 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:28,943 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:28:29,308 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:28:29,309 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:28:29,309 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:28:29,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:28:30,358 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:31,359 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:33,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:33,412 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:28:33,413 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:28:35,360 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:37,361 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:39,362 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:41,363 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:43,365 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:48,579 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:28:48,580 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:28:49,433 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:28:49,433 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:28:49,433 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:28:50,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:50,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:28:51,368 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:53,369 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:55,370 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:58,372 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:28:59,014 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:29:00,373 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:02,374 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:03,710 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:29:03,710 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:29:09,592 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:29:09,592 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:29:09,593 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:29:10,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:29:11,378 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:12,379 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:14,380 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:16,381 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:18,382 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:18,843 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:29:18,843 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:29:20,383 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:22,384 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:29,085 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:29:29,898 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:29:29,898 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:29:29,899 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:29:30,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:29:31,388 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:32,389 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:34,109 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:29:34,110 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:29:34,390 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:36,391 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:38,392 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:40,393 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:42,394 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:49,280 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:29:49,280 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:29:50,123 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:29:50,123 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:29:50,124 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:29:50,397 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:29:51,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:52,398 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:54,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:56,400 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:58,402 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:29:59,158 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:30:00,403 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:02,404 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:04,412 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:30:04,412 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:30:10,253 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:30:10,254 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:30:10,254 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:30:10,407 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:30:11,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:12,408 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:15,410 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:17,411 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:19,412 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:19,595 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:30:19,596 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:30:21,413 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:23,414 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:29,229 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:30:30,400 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:30:30,400 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:30:30,401 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:30:30,417 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:30:31,418 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:33,419 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:34,870 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:30:34,870 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:30:35,420 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:37,421 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:39,422 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:41,423 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:43,424 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:50,045 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:30:50,045 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:30:50,562 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:30:50,562 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:30:50,563 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:30:51,427 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:51,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:30:53,428 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:55,429 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:57,430 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:30:59,305 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:30:59,431 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:01,432 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:03,433 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:05,175 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:31:05,175 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:31:10,758 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:31:10,759 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:31:10,759 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:31:11,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:31:12,437 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:13,438 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:15,439 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:17,440 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:19,441 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:20,317 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:31:20,318 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:31:22,442 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:24,443 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:29,377 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:31:30,897 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:31:30,898 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:31:30,898 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:31:31,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:31:32,446 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:34,447 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:35,566 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:31:35,567 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:31:36,448 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:38,449 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:40,450 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:42,451 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:44,452 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:50,740 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:31:50,741 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:31:51,126 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:31:51,126 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:31:51,127 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:31:51,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:31:52,455 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:54,456 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:56,457 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:58,458 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:31:59,449 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:32:00,459 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:02,460 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:05,871 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:32:05,872 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:32:11,269 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:32:11,269 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:32:11,270 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:32:11,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:32:12,464 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:14,465 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:16,466 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:18,467 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:20,468 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:21,023 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:32:21,023 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:32:22,469 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:24,470 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:29,525 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:32:31,416 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:32:31,416 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:32:31,416 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:32:31,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:32:32,473 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:35,475 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:36,225 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:32:36,226 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:32:37,476 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:39,477 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:41,478 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:43,479 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:51,407 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:32:51,407 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:32:51,624 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:32:51,625 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:32:51,625 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:32:52,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:32:53,483 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:55,484 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:57,485 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:59,486 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:32:59,600 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:33:01,487 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:03,488 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:06,548 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:33:06,548 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:33:11,798 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:33:11,799 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:33:11,799 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:33:12,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:33:13,492 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:15,493 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:17,494 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:19,495 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:21,496 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:21,757 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:33:21,757 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:33:23,497 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:29,675 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:33:31,952 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:33:31,953 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:33:31,953 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:33:32,500 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:33:33,501 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:35,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:36,912 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:33:36,913 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:33:37,502 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:39,504 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:42,505 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:44,506 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:52,182 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:33:52,182 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:33:52,312 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:33:52,312 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:33:52,313 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:33:52,509 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:33:53,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:54,510 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:56,511 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:58,512 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:33:59,750 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:34:00,513 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:02,514 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:04,515 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:07,381 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:34:07,382 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:34:12,448 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:34:12,449 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:34:12,449 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:34:12,518 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:34:13,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:14,519 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:16,520 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:18,521 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:20,522 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:22,523 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:22,605 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:34:22,605 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:34:24,524 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:29,825 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:34:32,652 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:34:32,652 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:34:32,653 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:34:33,528 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:34:34,529 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:36,530 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:37,870 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:34:37,871 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:34:38,531 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:40,532 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:43,533 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:45,534 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:52,815 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:34:52,815 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:34:52,815 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:34:53,166 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:34:53,166 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:34:53,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:34:54,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:55,539 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:57,540 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:59,541 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:34:59,903 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:35:01,542 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:03,543 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:05,544 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:08,362 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:35:08,362 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:35:13,052 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:35:13,052 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:35:13,053 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:35:13,548 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:35:14,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:15,549 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:17,550 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:19,551 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:21,552 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:23,553 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:23,583 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:35:23,584 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:35:25,554 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:29,978 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:35:33,174 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:35:33,175 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:35:33,175 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:35:33,557 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:35:34,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:35,558 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:37,559 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:38,848 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:35:38,849 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:35:39,560 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:41,561 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:43,562 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:45,563 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:53,303 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:35:53,304 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:35:53,304 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:35:53,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:35:54,098 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:35:54,098 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:35:54,567 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:55,568 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:57,569 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:35:59,570 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:00,054 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:36:01,572 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:03,573 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:05,574 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:09,318 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:36:09,318 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:36:13,421 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:36:13,421 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:36:13,422 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:36:13,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:36:14,578 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:16,579 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:18,580 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:20,581 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:22,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:24,450 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:36:24,450 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:36:24,583 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:26,584 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:30,128 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:36:33,601 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:36:33,602 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:36:33,602 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:36:34,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:34,588 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:36:36,589 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:38,590 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:39,636 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:36:39,637 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:36:40,591 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:42,592 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:44,593 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:46,594 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:53,806 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:36:53,807 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:36:53,807 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:36:54,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:54,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:36:54,806 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:36:54,806 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:36:55,598 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:56,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:36:58,599 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:00,203 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:37:00,600 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:03,602 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:05,603 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:07,604 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:10,022 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:37:10,022 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:37:13,975 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:37:13,975 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:37:13,976 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:37:14,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:37:15,607 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:17,608 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:19,609 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:21,610 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:23,611 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:25,157 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:37:25,158 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:37:25,612 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:27,613 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:30,278 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:37:34,076 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:37:34,076 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:37:34,077 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:37:34,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:37:35,616 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:37,617 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:39,618 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:40,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:37:40,323 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:37:41,619 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:43,620 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:45,621 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:54,298 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:37:54,299 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:37:54,299 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:37:54,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:37:55,561 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:37:55,562 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:37:55,625 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:57,626 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:37:59,627 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:00,352 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:38:01,628 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:04,629 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:06,630 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:10,758 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:38:10,758 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:38:14,526 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:38:14,526 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:38:14,527 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:38:14,633 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:38:15,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:16,634 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:18,635 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:20,636 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:22,637 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:24,638 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:25,887 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:38:25,887 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:38:26,639 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:30,426 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:38:34,640 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:38:34,640 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:38:34,641 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:38:34,643 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:38:35,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:36,644 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:38,645 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:40,646 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:41,067 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:38:41,068 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:38:42,647 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:44,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:46,648 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:54,841 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:38:54,841 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:38:54,841 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:38:55,652 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:55,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:38:56,348 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:38:56,348 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:38:56,653 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:38:58,654 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:00,500 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:39:00,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:02,655 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:04,656 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:06,658 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:11,526 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:39:11,527 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:39:15,056 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:39:15,057 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:39:15,057 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:39:15,661 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:39:16,662 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:19,663 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:21,664 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:23,665 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:25,666 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:26,656 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:39:26,657 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:39:27,667 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:30,573 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:39:35,356 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:39:35,357 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:39:35,357 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:39:35,670 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:39:36,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:37,671 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:39,672 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:41,673 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:41,843 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:39:41,843 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:39:43,674 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:45,675 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:47,676 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:55,535 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:39:55,535 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:39:55,536 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:39:55,679 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:39:56,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:57,198 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:39:57,198 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:39:57,680 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:39:59,681 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:00,648 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:40:01,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:03,683 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:05,684 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:07,685 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:12,402 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:40:12,403 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:40:15,742 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:40:15,743 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:40:15,743 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:40:16,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:16,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:40:17,689 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:19,690 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:21,691 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:24,692 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:26,693 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:27,534 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:40:27,534 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:40:28,694 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:30,724 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:40:35,948 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:40:35,949 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:40:35,949 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:40:36,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:36,698 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:40:38,699 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:40,700 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:42,701 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:42,748 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:40:42,749 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:40:44,702 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:46,703 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:48,704 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:56,061 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:40:56,062 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:40:56,062 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:40:56,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:56,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:40:57,707 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:40:57,973 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:40:57,973 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:40:58,708 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:00,709 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:00,799 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:41:02,710 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:04,711 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:06,712 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:08,713 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:13,193 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:41:13,193 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:41:16,233 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:41:16,234 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:41:16,234 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:41:16,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:41:17,717 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:18,718 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:20,719 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:22,720 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:24,721 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:27,722 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:28,322 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:41:28,322 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:41:29,723 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:30,875 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:41:36,419 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:41:36,420 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:41:36,420 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:41:36,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:41:37,727 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:39,728 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:41,729 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:43,479 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:41:43,479 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:41:43,730 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:45,731 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:47,732 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:49,733 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:56,537 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:41:56,538 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:41:56,538 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:41:56,736 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:41:57,737 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:41:58,650 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:41:58,650 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:41:59,738 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:00,952 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:42:01,739 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:03,740 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:05,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:07,741 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:13,803 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:42:13,803 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:42:16,698 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:42:16,698 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:42:16,703 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:42:16,745 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:42:17,746 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:19,747 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:22,748 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:24,749 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:26,750 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:28,751 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:28,933 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:42:28,933 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:42:30,752 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:31,029 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:42:36,850 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:42:36,850 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:42:36,859 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:42:37,755 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:37,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:42:38,756 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:40,757 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:42,758 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:44,178 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:42:44,179 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:42:44,759 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:46,760 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:48,761 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:56,999 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:42:56,999 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:42:57,000 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:42:57,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:42:58,765 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:42:59,426 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:42:59,427 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:43:00,766 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:01,107 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:43:02,767 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:04,768 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:06,769 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:08,770 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:14,593 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:43:14,594 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:43:17,219 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:43:17,220 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:43:17,220 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:43:17,774 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:43:18,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:20,775 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:22,776 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:24,777 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:26,778 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:28,779 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:29,724 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:43:29,724 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:43:31,181 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:43:37,424 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:43:37,425 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:43:37,425 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:43:37,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:43:38,783 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:40,784 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:42,785 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:44,786 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:44,893 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:43:44,893 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:43:47,787 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:49,788 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:57,594 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:43:57,594 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:43:57,594 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:43:57,791 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:43:58,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:43:59,792 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:00,128 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:44:00,129 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:44:01,258 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:44:01,793 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:03,794 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:05,795 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:07,796 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:09,797 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:15,312 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:44:15,312 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:44:17,772 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:44:17,773 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:44:17,773 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:44:17,800 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:44:19,801 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:21,802 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:23,803 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:25,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:27,804 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:29,805 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:30,444 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:44:30,445 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:44:31,336 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:44:37,976 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:44:37,976 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:44:37,977 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:44:38,809 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:44:39,810 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:41,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:43,811 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:45,575 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:44:45,576 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:44:45,812 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:47,813 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:49,814 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:44:58,163 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:44:58,164 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:44:58,164 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:44:58,818 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:44:59,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:00,766 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:45:00,766 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:45:01,413 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:45:01,819 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:03,820 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:05,821 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:07,822 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:09,823 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:15,908 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:45:15,909 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:45:18,340 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:45:18,341 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:45:18,341 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:45:18,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:45:19,827 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:22,829 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:24,830 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:26,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:28,831 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:30,832 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:31,039 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:45:31,039 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:45:31,492 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:45:38,519 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:45:38,519 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:45:38,520 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:45:38,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:45:39,836 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:40,837 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:42,838 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:44,839 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:46,202 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:45:46,203 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:45:46,840 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:48,841 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:50,842 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:45:58,689 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:45:58,690 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:45:58,690 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:45:58,845 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:45:59,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:00,846 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:01,405 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:46:01,405 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:46:01,564 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:46:02,847 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:05,848 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:07,849 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:09,850 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:11,851 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:16,615 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:46:16,615 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:46:18,912 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:46:18,913 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:46:18,913 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:46:19,854 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:19,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:46:21,855 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:23,856 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:25,857 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:27,858 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:29,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:31,639 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:46:31,746 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:46:31,746 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:46:31,859 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:39,080 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:46:39,080 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:46:39,081 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:46:39,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:39,863 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:46:40,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:41,864 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:43,865 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:45,866 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:46,917 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:46:46,918 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:46:47,867 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:49,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:51,868 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:46:59,238 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:46:59,239 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:46:59,239 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:46:59,872 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:47:00,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:01,714 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:47:01,873 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:02,122 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:47:02,123 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:47:03,874 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:05,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:07,875 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:09,876 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:17,295 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:47:17,295 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:47:19,413 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:47:19,413 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:47:19,414 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:47:19,880 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:19,881 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:47:20,881 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:22,882 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:24,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:26,883 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:28,884 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:30,885 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:31,786 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:47:32,473 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:47:32,473 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:47:39,521 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:47:39,522 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:47:39,522 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:47:39,889 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:47:40,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:42,890 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:44,891 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:46,892 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:47,624 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:47:47,625 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:47:48,893 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:50,894 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:47:59,822 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:47:59,823 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:47:59,823 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:47:59,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:48:00,898 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:01,854 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:48:02,862 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:48:02,862 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:48:02,899 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:04,900 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:06,901 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:09,902 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:11,903 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:13,904 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:17,996 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:48:17,996 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:48:19,956 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:48:19,956 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:48:19,957 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:48:20,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:20,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:48:21,907 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:23,908 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:25,909 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:27,910 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:29,911 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:31,912 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:31,922 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:48:33,128 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:48:33,128 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:48:40,148 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:48:40,149 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:48:40,149 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:48:40,916 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:48:41,917 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:43,918 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:45,919 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:47,920 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:48,293 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:48:48,294 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:48:49,921 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:48:51,922 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:00,340 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:49:00,340 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:49:00,341 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:49:00,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:49:01,926 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:01,990 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:49:03,517 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:49:03,517 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:49:04,927 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:06,928 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:08,929 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:10,930 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:12,931 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:18,708 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:49:18,708 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:49:20,521 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:49:20,521 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:49:20,522 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:49:20,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:49:21,935 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:22,936 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:24,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:26,937 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:28,938 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:30,939 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:32,063 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:49:33,839 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:49:33,839 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:49:40,677 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:49:40,678 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:49:40,678 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:49:40,943 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:49:41,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:42,944 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:44,945 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:46,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:48,946 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:49:49,108 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:49:49,108 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:49:50,947 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:00,881 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:50:00,882 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:50:00,882 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:50:00,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:50:01,952 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:02,131 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:50:03,953 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:04,396 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:50:04,396 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:50:05,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:07,954 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:09,955 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:11,956 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:19,584 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:50:19,584 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:50:21,088 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:50:21,089 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:50:21,089 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:50:21,960 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:21,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:50:22,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:23,961 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:25,962 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:27,963 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:29,964 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:31,965 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:32,200 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:50:33,966 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:34,718 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:50:34,718 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:50:41,276 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:50:41,276 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:50:41,277 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:50:41,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:41,969 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:50:42,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:43,970 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:45,971 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:47,972 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:49,944 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:50:49,945 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:50:49,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:51,973 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:50:53,974 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:01,470 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:51:01,471 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:51:01,471 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:51:01,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:01,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:51:02,270 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:51:02,978 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:04,979 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:05,184 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:51:05,184 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:51:06,981 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:08,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:10,982 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:12,983 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:14,984 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:20,368 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:51:20,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:51:21,695 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:51:21,696 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:51:21,696 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:51:21,987 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:51:22,988 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:24,989 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:32,343 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:51:35,504 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:51:35,505 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:51:42,996 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:44,997 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:46,998 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:48,999 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:50,666 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:51:50,667 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:51:51,000 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:53,001 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:55,002 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:57,003 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:51:59,004 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:01,005 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:02,413 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:52:03,006 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:05,007 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:05,802 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:52:05,802 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:52:07,008 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:09,009 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:11,010 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:13,011 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:15,012 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:17,014 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:19,015 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:20,936 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:52:20,937 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:52:21,016 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:23,017 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:25,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:27,018 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:29,019 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:32,021 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:32,489 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:52:34,022 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:36,023 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:36,084 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:52:36,084 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:52:38,024 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:40,025 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:42,026 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:43,455 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:52:43,456 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:52:43,457 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:52:44,027 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:52:46,028 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:48,029 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:50,030 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:51,227 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:52:51,227 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:52:52,031 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:54,032 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:52:56,033 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:02,561 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:53:03,631 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:53:03,632 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:53:03,632 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:53:04,037 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:53:06,038 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:06,435 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:53:06,435 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:53:08,039 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:10,040 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:12,041 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:15,042 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:17,043 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:21,686 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:53:21,687 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:53:23,857 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:53:23,857 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:53:23,858 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:53:24,046 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:53:26,047 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:27,048 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:29,049 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:31,050 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:32,634 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:53:33,051 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:35,052 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:36,816 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:53:36,817 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:53:43,969 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:53:43,969 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:53:43,970 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:53:44,056 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:53:45,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:46,057 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:47,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:49,058 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:51,059 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:52,021 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:53:52,021 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:53:53,060 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:53:55,061 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:02,708 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:54:04,169 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:54:04,169 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:54:04,170 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:54:05,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:05,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:54:06,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:07,066 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:07,209 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:54:07,210 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:54:09,067 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:11,068 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:14,069 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:16,071 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:18,072 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:22,390 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:54:22,391 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:54:24,287 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:54:24,288 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:54:24,288 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:54:25,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:54:26,075 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:28,076 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:30,077 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:32,078 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:32,782 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:54:34,079 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:36,080 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:37,522 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:54:37,523 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:54:44,604 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:54:44,604 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:54:44,605 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:54:45,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:54:46,084 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:48,085 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:50,086 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:52,087 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:52,705 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:54:52,705 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:54:54,088 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:54:56,089 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:02,858 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:55:04,798 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:55:04,798 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:55:04,799 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:55:05,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:55:06,093 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:07,914 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:55:07,915 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:55:08,094 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:10,095 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:12,096 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:14,097 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:16,098 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:23,107 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:55:23,107 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:55:24,980 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:55:24,980 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:55:24,981 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:55:25,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:55:26,102 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:29,104 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:31,105 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:32,935 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:55:33,106 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:35,107 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:37,108 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:38,237 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:55:38,238 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:55:45,148 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:55:45,148 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:55:45,149 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:55:46,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:55:47,112 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:49,113 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:51,114 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:53,115 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:53,368 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:55:53,368 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:55:55,116 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:55:57,117 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:56:03,008 DEBUG SenderThread:799749 [sender.py:send():179] send: stats +2021-07-16 23:56:05,281 DEBUG SenderThread:799749 [sender.py:send():179] send: history +2021-07-16 23:56:05,282 DEBUG SenderThread:799749 [sender.py:send():179] send: summary +2021-07-16 23:56:05,282 INFO SenderThread:799749 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 23:56:06,120 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/wandb-summary.json +2021-07-16 23:56:07,121 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:56:08,857 DEBUG HandlerThread:799749 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 23:56:08,858 DEBUG SenderThread:799749 [sender.py:send_request():193] send_request: stop_status +2021-07-16 23:56:09,122 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:56:11,123 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:56:13,124 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log +2021-07-16 23:56:16,125 INFO Thread-8 :799749 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_223350-8eukt20m/files/output.log