diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -5920,3 +5920,5544 @@ 2023-11-18 16:32:28,186 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status 2023-11-18 16:32:28,187 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status 2023-11-18 16:32:30,759 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:32:33,228 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:32:39,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:32:40,819 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:32:43,184 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:32:43,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:32:43,186 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:32:44,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:32:49,905 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:32:50,888 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:32:51,726 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:32:55,727 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:32:58,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:32:58,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:32:58,185 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:32:58,945 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:33:01,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:07,434 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:09,026 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:33:12,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:14,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:33:14,186 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:33:14,187 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:33:17,481 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:21,096 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:33:21,727 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:33:22,727 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:27,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:29,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:33:29,536 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:33:29,540 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:33:33,176 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:33:33,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:39,451 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:43,250 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:33:44,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:33:44,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:33:44,495 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:33:44,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:49,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:51,727 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:33:53,310 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:33:54,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:33:59,495 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:33:59,495 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:33:59,496 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:34:00,710 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:01,359 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:06,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:09,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:34:09,956 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:34:09,957 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:34:09,960 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:34:10,414 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:34:10,416 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:11,420 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:11,962 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:14,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:34:14,495 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:34:17,668 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:21,510 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:21,727 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:34:22,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:27,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:30,846 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:34:30,906 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:34:30,907 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:34:32,595 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:33,114 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:38,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:42,659 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:43,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:45,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:34:45,846 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:34:49,041 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:34:50,724 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:34:51,728 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:34:54,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:00,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:00,786 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:35:00,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:35:00,846 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:35:05,455 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:08,848 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:35:11,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:16,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:16,892 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:35:16,893 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:35:16,895 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:35:18,934 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:35:21,453 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:21,729 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:35:26,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:31,008 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:35:31,895 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:35:31,896 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:35:31,896 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:35:32,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:37,458 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:41,067 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:35:42,819 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:46,989 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:35:46,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:35:46,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:35:47,983 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:49,120 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:35:51,729 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:35:53,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:58,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:35:59,197 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:01,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:36:01,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:36:04,236 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:09,299 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:09,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:14,913 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:16,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:36:16,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:36:19,369 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:20,223 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:21,730 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:36:25,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:29,435 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:31,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:31,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:36:31,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:36:36,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:39,492 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:41,706 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:46,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:36:46,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:36:47,248 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:47,547 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:51,730 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:36:52,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:36:57,612 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:36:58,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:01,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:37:01,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:37:04,159 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:07,683 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:37:09,654 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:14,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:16,991 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:37:16,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:37:17,746 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:37:20,240 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:21,731 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:37:25,732 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:25,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:37:25,847 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:37:25,847 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:37:25,849 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:37:26,808 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:37:26,809 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:37:30,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:31,991 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:37:31,991 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:37:36,526 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:36,876 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:37:41,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:44,935 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:37:47,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:37:47,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:37:47,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:37:47,316 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:51,732 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:37:52,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:37:55,015 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:37:57,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:02,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:38:02,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:38:02,020 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:38:02,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:05,092 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:38:08,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:13,153 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:38:13,906 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:17,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:38:17,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:38:17,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:38:19,234 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:21,750 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:38:23,315 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:38:24,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:29,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:32,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:38:32,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:38:32,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:38:33,389 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:38:35,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:40,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:43,449 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:38:45,821 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:47,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:38:47,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:38:47,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:38:51,208 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:38:51,736 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:38:53,508 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:38:56,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:01,568 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:39:01,826 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:02,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:39:02,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:39:02,020 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:39:07,200 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:11,631 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:39:12,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:17,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:39:17,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:39:17,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:39:18,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:21,690 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:39:21,744 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:39:23,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:28,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:31,884 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:39:32,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:39:32,019 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:39:32,020 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:39:34,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:40,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:41,941 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:39:45,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:47,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:39:47,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:39:47,381 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:39:48,989 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:39:50,457 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:39:51,734 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:39:55,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:00,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:02,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:40:02,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:40:03,076 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:40:06,570 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:11,183 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:40:11,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:16,761 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:17,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:40:17,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:40:21,735 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:40:22,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:23,240 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:40:28,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:31,299 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:40:32,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:40:32,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:40:33,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:38,876 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:40,156 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:40:40,156 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:40:40,157 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:40:40,159 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:40:40,357 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:40:41,363 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:40:44,161 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:48,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:40:48,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:40:48,290 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:40:49,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:51,456 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:40:51,735 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:40:54,643 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:40:59,520 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:41:00,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:03,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:41:03,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:41:03,289 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:41:05,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:10,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:11,601 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:41:15,645 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:18,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:41:18,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:41:18,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:41:21,451 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:21,681 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:41:21,735 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:41:26,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:29,741 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:41:31,703 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:33,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:41:33,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:41:33,289 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:41:37,468 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:39,812 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:41:42,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:47,646 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:48,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:41:48,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:41:48,667 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:41:51,736 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:41:51,900 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:41:52,737 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:41:58,645 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:00,043 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:42:03,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:42:03,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:42:03,611 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:42:03,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:09,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:10,109 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:42:14,645 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:18,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:42:18,611 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:42:18,611 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:42:19,823 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:20,179 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:42:21,737 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:42:25,739 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:29,270 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:42:31,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:33,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:42:33,613 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:42:33,615 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:42:36,758 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:37,331 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:42:42,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:47,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:48,611 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:42:48,667 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:42:48,669 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:42:51,737 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:42:52,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:42:53,444 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:42:58,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:03,505 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:43:03,611 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:43:03,618 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:43:03,619 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:43:03,801 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:08,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:13,654 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:43:14,598 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:19,599 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:19,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:43:19,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:43:19,667 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:43:21,738 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:43:21,933 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:43:24,739 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:30,709 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:32,017 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:43:34,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:43:34,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:43:34,666 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:43:35,903 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:40,097 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:43:41,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:46,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:49,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:43:49,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:43:49,676 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:43:51,738 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:43:51,739 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:52,184 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:43:57,171 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:43:59,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:43:59,457 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:43:59,457 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:43:59,459 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:44:00,258 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:44:01,262 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:02,265 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:02,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:04,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:44:04,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:44:04,676 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:44:07,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:10,323 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:13,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:18,171 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:19,390 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:19,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:44:19,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:44:19,676 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:44:21,739 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:44:23,740 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:29,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:29,463 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:34,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:35,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:44:35,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:44:35,381 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:44:37,529 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:39,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:45,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:49,626 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:44:50,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:44:50,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:44:50,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:44:50,562 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:51,739 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:44:55,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:44:57,687 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:01,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:05,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:45:05,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:45:05,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:45:05,748 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:07,540 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:13,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:15,808 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:19,007 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:20,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:45:20,381 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:45:20,383 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:45:21,741 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:45:23,860 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:24,930 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:30,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:33,918 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:35,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:45:35,380 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:45:35,380 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:45:35,549 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:40,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:43,993 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:45,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:50,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:45:51,017 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:45:51,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:45:51,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:45:51,742 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:45:54,083 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:45:56,154 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:01,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:02,129 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:46:06,017 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:46:06,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:46:06,018 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:46:07,203 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:12,191 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:46:12,927 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:17,929 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:21,017 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:46:21,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:46:21,018 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:46:21,753 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:46:22,246 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:46:23,754 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:29,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:32,312 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:46:34,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:36,017 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:46:36,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:46:36,019 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:46:40,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:42,381 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:46:45,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:50,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:46:51,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:46:51,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:46:51,018 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:46:51,742 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:46:52,449 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:46:55,744 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:00,504 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:01,453 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:06,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:47:06,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:47:06,338 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:47:06,566 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:07,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:47:07,365 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:47:07,366 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:47:07,368 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:47:07,559 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:47:08,564 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:09,568 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:12,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:17,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:17,630 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:21,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:47:21,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:47:21,338 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:47:21,743 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:47:22,744 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:27,688 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:27,797 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:33,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:36,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:47:36,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:47:36,386 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:47:37,762 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:38,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:44,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:47,826 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:49,985 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:47:51,337 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:47:51,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:47:51,338 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:47:51,743 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:47:55,881 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:47:55,984 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:01,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:05,951 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:48:06,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:48:06,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:48:06,338 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:48:06,497 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:11,499 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:16,027 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:48:16,504 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:21,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:48:21,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:48:21,339 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:48:21,508 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:21,744 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:48:26,090 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:48:27,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:32,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:36,165 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:48:36,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:48:36,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:48:36,339 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:48:37,561 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:42,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:46,237 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:48:48,058 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:51,747 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:48:52,470 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:48:52,471 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:48:52,473 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:48:53,646 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:48:56,315 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:48:59,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:04,376 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:49:05,219 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:07,470 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:49:07,471 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:49:07,471 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:49:10,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:14,497 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:49:16,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:21,418 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:21,748 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:49:22,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:49:22,656 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:49:22,658 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:49:26,586 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:49:26,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:32,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:34,638 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:49:37,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:49:37,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:49:37,609 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:49:37,803 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:43,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:44,700 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:49:48,418 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:51,748 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:49:52,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:49:52,609 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:49:52,610 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:49:52,755 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:49:53,816 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:49:58,817 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:01,813 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:04,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:08,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:50:08,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:50:08,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:50:09,866 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:10,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:16,094 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:19,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:50:19,826 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:50:19,827 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:50:19,829 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:50:19,937 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:50:19,938 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:21,749 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:50:21,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:21,948 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:23,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:50:23,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:50:23,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:50:27,280 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:30,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:33,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:38,066 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:38,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:50:38,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:50:38,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:50:38,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:43,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:48,130 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:49,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:51,749 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:50:53,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:50:53,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:50:53,119 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:50:54,355 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:50:58,209 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:50:59,980 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:05,228 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:08,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:51:08,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:51:08,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:51:08,289 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:51:10,361 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:16,227 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:16,350 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:51:21,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:21,751 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:51:23,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:51:23,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:51:23,167 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:51:26,410 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:51:26,934 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:32,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:34,490 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:51:38,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:38,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:51:38,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:51:38,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:51:43,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:44,565 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:51:48,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:51,751 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:51:52,621 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:51:53,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:51:53,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:51:53,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:51:54,309 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:51:59,310 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:02,692 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:04,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:08,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:52:08,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:52:08,168 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:52:10,413 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:12,754 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:16,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:21,048 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:21,751 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:52:22,819 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:23,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:52:23,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:52:23,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:52:26,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:30,873 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:31,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:36,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:38,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:52:38,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:52:38,168 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:52:40,987 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:41,964 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:46,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:50,053 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:51,752 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:52:52,752 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:52:53,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:52:53,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:52:53,119 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:52:58,092 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:52:58,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:04,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:08,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:53:08,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:53:08,166 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:53:09,383 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:10,163 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:14,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:18,372 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:19,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:21,752 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:53:23,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:53:23,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:53:23,119 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:53:25,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:26,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:53:26,427 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:53:26,428 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:53:26,432 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:53:26,486 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:53:27,491 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:28,496 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:30,435 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:36,382 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:36,556 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:38,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:53:38,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:53:38,119 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:53:42,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:48,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:48,688 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:51,753 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:53:53,119 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:53:53,176 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:53:53,177 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:53:53,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:53:56,739 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:53:58,866 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:04,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:08,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:54:08,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:54:08,247 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:54:09,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:10,831 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:54:14,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:19,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:20,900 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:54:21,753 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:54:23,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:54:23,193 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:54:25,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:30,457 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:31,009 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:54:36,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:38,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:54:38,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:54:38,194 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:54:39,071 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:54:41,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:46,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:50,316 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:54:51,753 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:54:51,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:53,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:54:53,194 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:54:57,410 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:54:58,382 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:03,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:08,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:55:08,194 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:55:08,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:08,458 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:13,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:18,539 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:19,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:21,754 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:55:24,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:55:24,706 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:55:24,708 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:55:24,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:26,635 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:30,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:36,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:36,696 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:39,694 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:55:39,696 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:55:39,697 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:55:41,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:46,909 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:48,779 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:51,755 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:55:52,756 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:55:54,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:55:54,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:55:54,957 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:55:56,838 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:55:58,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:03,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:06,905 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:08,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:09,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:56:09,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:56:09,908 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:56:14,081 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:16,985 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:19,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:21,755 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:56:24,757 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:24,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:56:24,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:56:24,957 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:56:27,051 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:30,629 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:36,429 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:37,109 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:39,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:56:39,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:56:39,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:56:42,119 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:43,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 16:56:43,439 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 16:56:43,439 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 16:56:43,442 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 16:56:44,154 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 16:56:45,158 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:46,161 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:47,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:51,757 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:56:52,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:56:54,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:56:54,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:56:54,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:56:55,215 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:56:58,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:03,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:05,277 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:57:08,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:09,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:57:09,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:57:09,956 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:57:14,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:15,355 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:57:19,455 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:21,758 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:57:24,420 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:57:24,605 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:24,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:57:24,909 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:57:24,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:57:30,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:32,471 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:57:35,456 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:39,909 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:57:39,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:57:39,957 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:57:40,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:42,538 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:57:45,750 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:50,587 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:57:51,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:57:51,758 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:57:54,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:57:54,909 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:57:54,909 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:57:57,143 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:02,530 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:02,690 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:58:07,600 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:09,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:58:09,926 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:58:10,759 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:58:13,172 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:18,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:20,821 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:58:21,758 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:58:23,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:24,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:58:24,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:58:24,926 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:58:28,874 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:58:29,212 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:34,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:38,950 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:58:39,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:58:39,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:58:39,993 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:58:40,239 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:45,241 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:49,032 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:58:50,599 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:58:51,759 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:58:54,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:58:54,992 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:58:56,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:01,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:05,169 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:59:06,605 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:09,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:59:09,993 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:59:12,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:17,327 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:59:17,330 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:21,759 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:59:22,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:24,993 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:59:24,994 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:59:25,036 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:59:27,404 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:59:28,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:34,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:37,468 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:59:39,881 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:39,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:59:39,993 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:59:45,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:45,529 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 16:59:50,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:51,761 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 16:59:54,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 16:59:54,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 16:59:54,995 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 16:59:56,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 16:59:57,601 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:01,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:03,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:00:03,876 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:00:03,877 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:00:03,878 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:00:04,646 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:00:04,647 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:06,654 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:06,880 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:09,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:00:09,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:00:09,997 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:00:12,179 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:14,706 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:17,836 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:21,762 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:00:22,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:23,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:24,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:00:24,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:00:24,995 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:00:29,269 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:32,851 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:34,742 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:39,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:39,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:00:40,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:00:40,036 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:00:42,919 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:45,553 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:50,753 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:00:51,762 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:00:52,981 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:00:54,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:00:54,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:00:54,995 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:00:56,198 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:01,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:03,049 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:01:06,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:09,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:01:09,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:01:09,995 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:01:11,106 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:01:12,182 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:17,184 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:21,182 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:01:21,762 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:01:22,763 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:24,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:01:24,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:01:24,995 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:01:28,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:31,267 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:01:33,742 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:38,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:39,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:01:39,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:01:39,996 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:01:41,334 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:01:44,207 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:49,747 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:51,620 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:01:51,763 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:01:54,764 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:01:54,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:01:55,046 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:01:55,048 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:01:59,680 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:02:00,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:05,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:09,746 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:02:09,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:02:09,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:02:09,996 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:02:11,251 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:16,360 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:16,798 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:02:21,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:21,766 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:02:24,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:02:24,997 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:02:26,871 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:02:27,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:32,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:36,938 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:02:38,230 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:39,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:02:39,997 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:02:44,208 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:47,009 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:02:50,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:51,765 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:02:54,996 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:02:54,997 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:02:55,243 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:02:57,074 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:00,462 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:06,171 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:07,153 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:09,997 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:03:09,997 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:03:11,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:16,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:17,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:03:17,224 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:03:17,224 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:03:17,227 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:03:18,232 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:03:18,233 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:19,237 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:21,766 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:03:21,766 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:25,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:03:25,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:03:25,308 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:03:27,276 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:27,286 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:32,278 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:37,358 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:38,178 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:40,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:03:40,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:03:40,306 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:03:43,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:45,419 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:03:48,979 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:51,768 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:03:54,766 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:03:55,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:03:55,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:03:55,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:03:55,486 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:00,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:05,565 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:06,178 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:10,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:04:10,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:04:10,306 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:04:11,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:15,635 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:16,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:21,768 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:04:22,770 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:25,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:04:25,261 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:04:25,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:04:25,695 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:28,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:33,701 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:33,764 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:35,773 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:37,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:39,180 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:41,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:04:41,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:04:41,168 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:04:44,330 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:44,978 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:49,331 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:51,769 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:04:53,035 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:04:54,770 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:04:56,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:04:56,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:05:00,230 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:01,086 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:05:05,231 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:10,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:11,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:05:11,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:05:15,451 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:17,172 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:05:21,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:21,772 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:05:25,233 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:05:26,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:05:26,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:05:26,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:05:26,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:31,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:35,297 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:05:37,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:42,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:05:42,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:42,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:05:42,336 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:05:45,376 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:05:47,437 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:51,772 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:05:52,703 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:05:55,447 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:05:57,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:05:57,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:05:57,284 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:05:58,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:03,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:05,531 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:08,636 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:12,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:06:12,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:06:12,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:06:14,325 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:15,609 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:19,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:21,773 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:06:23,663 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:24,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:27,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:06:27,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:06:27,284 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:06:30,486 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:31,710 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:06:31,720 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:06:31,720 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:06:31,723 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:06:31,730 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:06:32,734 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:33,737 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:35,726 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:40,766 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:42,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:06:42,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:06:42,284 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:06:43,807 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:46,432 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:51,636 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:51,773 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:06:53,865 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:06:56,774 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:06:57,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:06:57,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:06:57,337 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:07:01,933 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:07:02,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:07,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:12,004 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:07:12,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:07:12,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:07:12,284 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:07:13,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:18,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:21,774 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:07:22,074 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:07:23,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:27,284 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:07:27,325 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:07:27,326 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:07:28,847 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:30,131 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:07:33,848 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:39,074 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:39,203 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:07:42,284 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:07:42,284 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:07:42,285 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:07:44,519 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:49,269 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:07:50,373 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:51,774 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:07:55,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:07:57,284 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:07:57,284 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:07:57,285 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:07:57,324 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:08:01,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:06,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:07,387 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:08:11,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:13,656 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:08:13,706 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:08:13,708 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:08:16,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:17,453 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:08:21,775 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:08:22,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:27,535 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:08:27,778 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:28,699 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:08:28,700 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:08:28,700 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:08:32,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:38,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:41,629 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:08:43,699 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:08:43,700 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:08:43,700 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:08:43,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:48,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:51,693 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:08:51,775 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:08:54,777 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:08:59,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:08:59,316 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:08:59,317 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:09:00,453 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:01,773 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:05,454 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:10,605 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:11,858 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:14,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:09:14,265 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:09:16,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:21,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:21,776 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:09:21,938 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:26,777 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:30,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:09:30,686 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:09:31,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:34,022 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:37,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:42,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:44,106 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:45,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:09:45,687 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:09:47,852 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:51,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:09:51,549 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:09:51,550 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:09:51,553 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:09:51,776 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:09:52,162 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:09:53,167 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:53,778 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:09:54,170 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:09:58,786 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:00,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:10:00,689 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:10:00,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:10:04,251 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:10:04,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:09,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:14,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:15,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:10:15,687 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:10:19,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:21,777 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:10:25,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:25,430 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:10:30,179 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:31,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:10:31,971 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:10:31,973 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:10:35,507 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:10:36,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:41,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:45,586 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:10:46,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:46,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:10:46,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:10:46,970 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:10:51,778 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:10:51,779 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:57,087 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:10:57,670 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:02,089 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:03,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:11:03,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:11:03,223 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:11:05,726 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:07,661 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:12,663 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:15,797 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:18,201 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:18,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:11:18,222 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:11:21,779 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:11:23,858 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:23,939 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:28,940 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:33,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:11:33,222 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:11:33,924 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:34,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:39,662 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:43,985 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:45,293 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:49,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:11:49,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:11:49,549 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:11:50,601 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:11:51,780 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:11:52,052 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:11:55,781 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:01,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:02,118 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:12:04,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:12:04,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:12:04,547 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:12:06,764 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:10,172 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:12:12,401 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:18,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:19,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:12:19,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:12:19,547 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:12:20,239 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:12:21,780 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:12:23,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:28,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:30,307 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:12:33,686 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:34,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:12:34,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:12:34,547 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:12:39,482 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:40,393 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:12:44,483 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:49,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:12:49,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:12:49,548 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:12:49,801 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:50,456 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:12:51,781 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:12:55,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:12:58,542 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:01,144 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:05,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:13:05,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:13:05,321 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:13:06,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:08,605 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:11,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:16,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:17,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:13:17,216 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:13:17,216 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:13:17,219 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:13:17,711 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:13:17,712 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:19,719 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:20,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:13:20,319 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:13:21,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:21,781 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:13:27,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:27,777 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:32,205 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:35,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:13:35,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:13:35,319 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:13:35,825 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:37,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:42,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:45,890 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:13:48,333 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:50,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:13:50,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:13:50,319 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:13:51,782 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:13:53,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:13:58,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:01,984 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:14:03,595 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:05,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:14:05,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:14:05,320 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:14:09,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:12,056 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:14:14,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:19,927 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:20,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:14:20,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:14:20,320 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:14:21,782 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:14:22,144 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:14:25,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:30,946 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:32,225 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:14:35,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:14:35,319 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:14:36,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:40,290 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:14:41,897 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:46,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:50,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:14:50,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:14:50,320 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:14:51,784 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:14:52,786 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:14:56,403 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:14:58,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:03,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:05,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:15:05,320 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:15:06,465 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:15:08,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:13,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:14,525 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:15:18,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:20,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:15:20,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:15:20,858 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:15:21,784 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:15:24,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:24,596 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:15:29,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:34,654 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:15:34,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:35,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:15:35,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:15:35,856 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:15:40,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:44,721 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:15:46,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:50,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:15:50,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:15:50,857 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:15:51,785 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:15:51,785 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:15:52,771 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:15:57,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:00,830 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:03,166 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:06,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:16:06,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:16:06,927 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:16:08,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:09,887 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:13,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:19,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:19,949 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:21,785 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:16:21,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:16:21,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:16:21,927 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:16:24,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:28,003 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:29,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:34,972 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:35,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:16:35,910 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:16:35,910 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:16:35,913 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:16:36,061 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:16:36,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:16:36,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:16:36,927 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:16:37,066 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:38,070 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:40,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:46,133 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:46,558 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:51,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:16:51,786 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:16:52,849 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:16:52,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:16:52,851 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:16:56,205 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:16:56,839 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:01,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:04,263 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:17:07,466 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:07,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:17:07,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:17:07,850 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:17:12,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:14,332 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:17:17,712 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:21,787 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:17:22,788 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:22,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:17:22,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:17:22,858 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:17:24,396 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:17:28,580 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:32,454 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:17:33,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:38,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:17:38,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:17:38,492 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:17:38,707 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:40,506 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:17:44,522 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:50,463 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:17:50,565 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:17:51,788 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:17:53,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:17:53,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:17:53,492 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:17:55,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:00,638 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:18:01,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:06,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:08,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:18:08,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:18:08,492 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:18:10,707 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:18:11,742 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:16,744 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:20,772 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:18:21,788 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:18:21,789 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:23,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:18:23,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:18:23,494 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:18:27,353 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:28,851 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:18:32,353 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:37,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:38,495 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:18:38,495 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:18:38,496 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:18:38,929 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:18:43,757 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:46,990 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:18:49,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:51,789 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:18:53,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:18:53,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:18:53,495 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:18:54,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:18:59,064 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:00,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:05,135 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:06,117 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:08,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:19:08,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:19:08,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:19:11,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:16,124 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:16,186 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:21,126 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:21,791 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:19:23,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:19:23,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:19:23,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:19:24,240 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:27,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:32,093 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:34,305 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:37,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:38,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:19:38,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:19:38,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:19:41,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:19:41,757 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:19:41,757 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:19:41,761 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:19:42,364 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:19:42,365 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:42,762 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:43,368 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:47,763 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:51,805 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:19:53,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:19:53,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:19:53,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:19:53,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:19:54,434 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:19:58,713 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:02,491 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:20:04,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:08,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:20:08,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:20:08,496 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:20:09,782 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:12,673 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:20:15,437 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:20,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:21,793 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:20:22,746 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:20:23,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:20:23,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:20:23,494 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:20:25,669 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:30,714 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:32,828 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:20:35,864 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:38,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:20:38,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:20:38,494 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:20:41,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:44,909 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:20:46,985 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:51,794 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:20:52,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:20:52,970 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:20:53,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:20:53,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:20:53,494 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:20:58,733 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:03,042 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:21:03,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:08,949 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:09,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:21:09,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:21:09,301 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:21:13,099 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:21:14,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:19,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:21,794 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:21:23,168 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:21:24,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:21:24,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:21:24,300 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:21:25,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:31,186 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:31,216 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:21:36,187 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:39,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:21:39,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:21:39,301 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:21:41,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:43,291 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:21:46,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:50,346 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:21:51,794 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:21:51,795 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:21:54,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:21:54,301 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:21:57,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:00,417 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:22:03,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:09,081 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:09,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:22:09,300 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:22:10,488 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:22:14,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:18,550 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:22:20,457 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:21,795 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:22:24,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:22:24,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:22:24,301 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:22:25,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:30,645 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:22:30,711 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:35,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:39,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:22:39,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:22:39,300 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:22:40,713 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:22:40,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:45,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:50,803 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:22:51,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:51,796 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:22:54,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:22:54,301 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:22:56,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:22:58,165 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:22:58,166 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:22:58,167 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:22:58,169 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:22:58,870 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:22:59,874 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:00,878 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:02,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:07,171 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:09,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:23:09,303 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:23:12,946 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:13,036 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:18,037 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:21,799 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:23:23,502 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:24,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:23:24,301 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:23:25,025 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:28,967 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:33,075 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:34,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:39,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:39,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:23:39,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:23:39,639 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:23:43,141 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:44,966 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:50,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:23:51,193 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:23:51,798 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:23:54,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:23:54,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:23:54,639 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:23:55,818 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:00,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:01,263 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:24:05,889 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:09,382 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:24:09,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:24:09,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:24:09,639 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:24:11,882 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:17,864 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:21,440 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:24:21,798 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:24:23,799 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:24,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:24:24,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:24:24,643 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:24:28,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:29,499 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:24:33,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:38,569 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:24:39,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:39,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:24:39,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:24:39,639 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:24:44,864 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:46,620 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:24:49,889 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:51,799 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:24:54,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:24:54,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:24:54,640 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:24:55,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:24:56,687 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:00,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:04,739 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:06,068 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:09,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:25:09,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:25:09,640 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:25:11,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:16,821 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:17,750 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:21,801 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:25:22,803 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:24,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:25:24,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:25:24,640 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:25:24,877 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:27,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:33,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:34,942 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:38,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:40,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:25:40,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:25:40,858 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:25:43,014 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:44,792 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:49,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:51,801 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:25:53,092 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:25:54,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:25:55,855 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:25:55,856 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:26:00,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:03,158 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:05,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:10,495 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:10,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:26:10,857 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:26:11,300 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:26:11,302 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:26:11,303 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:26:11,307 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:26:12,226 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:26:12,227 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:13,236 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:16,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:21,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:21,801 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:26:23,300 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:25,855 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:26:25,856 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:26:27,076 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:31,354 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:32,506 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:37,507 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:40,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:26:40,856 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:26:41,455 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:43,069 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:48,867 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:49,517 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:26:51,802 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:26:54,803 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:26:56,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:26:56,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:26:56,447 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:26:59,592 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:27:00,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:05,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:09,663 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:27:11,187 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:11,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:27:11,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:27:11,446 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:27:16,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:19,755 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:27:21,802 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:27:21,803 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:26,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:27:26,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:27:26,449 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:27:27,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:28,824 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:27:33,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:38,922 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:27:39,395 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:41,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:27:41,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:27:41,447 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:27:44,614 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:49,010 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:27:49,650 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:51,803 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:27:54,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:27:56,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:27:56,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:27:56,447 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:27:57,068 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:00,640 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:05,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:09,147 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:11,144 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:11,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:28:11,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:28:11,447 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:28:16,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:19,211 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:21,688 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:21,803 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:28:26,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:28:26,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:28:26,447 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:28:27,262 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:27,620 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:32,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:37,327 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:38,038 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:41,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:28:41,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:28:41,451 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:28:43,706 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:45,391 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:49,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:51,806 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:28:55,266 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:28:55,469 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:28:56,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:28:56,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:28:56,448 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:29:00,659 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:05,550 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:05,754 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:10,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:12,079 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:29:12,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:29:12,082 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:29:15,621 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:16,659 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:21,661 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:21,807 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:29:22,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:29:22,346 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:29:22,347 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:29:22,348 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:29:22,701 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:29:23,707 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:24,710 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:27,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:29:27,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:29:27,081 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:29:27,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:33,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:33,770 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:38,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:41,834 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:42,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:29:42,080 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:29:42,081 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:29:43,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:48,661 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:51,807 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:29:51,910 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:29:53,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:29:57,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:29:57,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:29:57,913 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:29:59,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:00,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:30:04,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:09,661 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:11,049 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:30:12,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:30:12,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:30:12,912 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:30:15,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:20,879 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:21,120 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:30:21,835 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:30:26,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:27,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:30:27,913 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:30:27,914 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:30:31,207 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:30:32,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:37,662 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:39,268 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:30:42,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:30:42,913 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:30:42,914 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:30:43,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:48,343 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:49,343 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:30:51,823 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:30:53,824 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:57,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:30:57,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:30:57,913 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:30:59,100 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:30:59,420 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:05,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:09,487 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:10,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:12,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:31:12,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:31:12,913 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:31:16,079 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:19,556 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:21,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:21,824 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:31:26,684 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:27,616 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:27,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:31:27,913 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:31:27,913 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:31:32,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:37,676 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:37,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:42,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:42,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:31:42,913 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:31:42,913 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:31:47,750 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:48,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:51,824 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:31:53,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:31:57,832 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:31:57,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:31:57,913 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:31:57,914 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:31:59,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:05,067 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:07,917 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:32:10,235 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:12,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:32:12,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:32:12,919 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:32:15,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:15,980 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:32:21,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:21,827 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:32:24,040 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:32:26,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:27,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:32:27,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:32:27,918 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:32:32,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:37,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:42,067 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:42,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:32:42,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:32:42,967 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:32:48,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:51,827 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:32:53,828 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:32:56,435 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:32:56,436 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:32:56,436 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:32:56,439 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:32:57,227 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:32:57,228 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:32:57,917 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:32:57,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:32:57,919 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:32:58,233 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:32:59,105 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:05,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:07,308 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:33:10,378 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:12,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:33:12,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:33:12,921 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:33:15,839 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:17,372 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:33:21,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:21,827 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:33:25,431 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:33:26,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:27,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:33:27,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:33:27,919 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:33:32,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:35,498 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:33:37,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:42,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:42,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:33:42,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:33:42,920 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:33:45,605 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:33:47,419 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:51,830 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:33:52,833 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:33:55,670 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:33:57,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:33:57,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:33:57,919 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:33:58,167 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:03,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:05,789 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:34:08,864 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:12,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:34:12,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:34:12,921 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:34:14,372 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:15,864 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:34:20,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:21,831 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:34:25,127 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:25,943 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:34:27,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:34:27,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:34:27,920 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:34:31,097 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:34,003 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:34:37,032 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:42,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:42,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:34:42,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:34:42,920 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:34:44,067 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:34:47,147 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:51,832 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:34:52,835 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:34:56,168 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:34:57,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:34:57,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:34:57,920 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:34:58,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:03,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:06,252 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:35:08,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:12,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:35:12,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:35:12,921 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:35:14,133 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:14,326 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:35:20,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:21,833 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:35:24,402 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:35:25,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:27,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:35:27,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:35:27,920 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:35:31,093 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:34,461 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:35:37,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:42,034 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:42,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:35:42,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:35:42,921 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:35:43,535 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:35:47,113 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:51,835 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:35:52,837 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:35:55,710 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:35:57,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:35:57,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:35:57,921 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:35:58,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:03,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:05,785 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:08,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:12,921 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:36:12,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:36:12,926 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:36:13,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:13,954 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:36:13,955 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:36:13,956 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:36:13,959 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:36:14,857 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:36:14,858 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:15,863 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:19,178 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:21,837 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:36:24,801 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:25,944 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:27,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:36:27,921 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:36:27,921 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:36:30,121 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:34,009 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:35,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:40,198 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:43,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:36:43,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:36:43,516 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:36:45,682 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:46,078 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:51,179 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:51,838 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:36:56,146 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:36:56,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:36:58,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:36:58,515 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:37:01,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:06,218 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:37:07,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:12,036 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:13,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:37:13,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:37:13,516 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:37:16,284 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:37:17,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:21,839 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:37:22,842 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:24,342 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:37:28,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:28,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:37:28,516 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:37:33,766 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:36,434 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:37:39,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:43,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:37:43,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:37:43,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:37:44,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:46,504 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:37:49,872 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:51,839 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:37:55,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:37:58,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:37:58,576 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:37:58,577 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:38:01,776 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:02,612 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:38:06,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:12,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:12,679 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:38:13,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:38:13,516 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:38:17,683 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:21,732 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:38:21,840 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:38:22,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:27,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:28,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:38:28,516 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:38:33,718 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:38,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:41,883 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:38:43,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:38:43,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:38:43,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:38:44,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:49,723 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:51,841 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:38:51,959 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:38:54,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:38:59,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:38:59,688 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:38:59,689 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:38:59,843 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:02,040 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:04,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:09,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:12,109 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:14,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:39:14,688 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:39:15,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:20,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:21,841 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:39:22,185 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:26,842 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:31,854 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:32,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:39:32,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:39:32,151 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:39:34,513 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:36,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:41,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:46,703 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:47,150 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:39:47,150 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:39:47,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:51,843 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:39:52,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:39:53,855 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:39:53,856 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:39:53,857 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:39:53,859 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:39:54,768 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:39:54,770 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:56,776 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:39:57,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:02,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:40:02,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:40:02,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:40:03,518 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:05,943 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:40:08,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:14,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:16,029 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:40:17,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:40:17,340 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:40:19,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:21,843 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:40:25,414 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:26,098 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:40:30,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:32,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:40:32,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:40:32,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:40:36,175 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:40:36,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:41,176 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:46,240 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:40:46,979 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:47,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:40:47,341 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:40:51,845 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:40:52,847 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:40:56,309 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:40:58,708 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:03,244 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:41:03,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:41:03,246 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:41:04,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:06,391 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:41:09,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:15,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:16,462 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:41:18,244 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:41:18,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:41:18,245 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:41:20,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:21,845 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:41:26,538 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:41:26,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:31,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:34,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:41:34,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:41:34,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:41:36,609 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:41:37,418 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:42,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:46,673 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:41:47,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:49,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:41:49,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:41:49,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:41:51,846 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:41:53,848 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:41:56,741 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:41:59,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:04,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:42:04,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:42:04,254 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:42:04,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:06,808 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:42:09,958 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:14,877 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:42:15,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:19,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:42:19,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:42:19,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:42:20,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:21,848 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:42:26,629 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:26,966 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:42:31,631 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:34,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:42:34,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:42:34,256 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:42:36,203 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:42:36,958 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:41,959 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:46,281 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:42:47,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:49,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:42:49,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:42:51,849 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:42:52,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:42:56,352 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:42:58,839 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:04,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:43:04,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:43:04,423 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:43:04,463 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:09,958 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:14,960 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:18,325 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:43:18,348 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:43:18,349 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:43:18,350 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:43:18,517 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:43:19,254 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:43:19,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:43:19,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:43:19,523 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:43:20,528 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:43:20,958 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:21,849 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:43:26,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:28,795 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:43:32,698 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:34,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:43:34,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:43:34,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:43:38,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:38,862 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:43:43,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:48,942 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:43:48,962 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:49,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:43:49,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:43:49,255 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:43:51,849 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:43:54,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:43:59,012 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:00,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:05,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:44:05,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:44:05,121 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:44:06,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:09,073 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:11,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:16,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:19,133 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:20,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:44:20,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:44:20,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:44:21,850 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:44:21,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:26,852 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:28,208 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:32,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:35,570 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:44:35,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:44:35,572 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:44:37,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:38,282 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:42,675 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:48,030 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:48,343 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:50,570 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:44:50,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:44:50,571 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:44:51,850 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:44:53,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:44:56,413 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:44:59,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:04,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:05,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:45:05,573 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:45:05,574 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:45:08,500 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:45:09,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:14,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:16,563 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:45:19,985 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:20,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:45:20,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:45:20,572 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:45:21,851 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:45:25,490 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:26,625 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:45:30,491 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:35,493 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:35,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:45:35,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:45:35,573 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:45:36,702 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:45:41,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:46,665 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:46,774 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:45:50,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:45:50,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:45:50,572 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:45:51,852 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:45:51,853 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:45:56,839 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:45:57,083 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:02,084 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:05,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:46:05,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:46:05,709 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:46:07,978 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:08,927 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:46:13,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:18,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:19,065 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:46:20,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:46:20,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:46:20,671 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:46:21,852 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:46:23,853 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:29,142 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:46:29,160 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:34,162 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:35,459 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:46:35,463 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:46:35,464 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:46:35,467 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:46:35,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:46:35,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:46:35,671 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:46:36,194 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:46:37,198 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:46:39,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:44,406 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:47,313 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:46:49,737 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:50,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:46:50,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:46:50,671 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:46:51,852 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:46:54,853 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:46:59,855 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:00,404 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:47:05,040 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:05,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:47:05,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:47:05,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:47:10,479 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:47:11,000 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:16,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:20,547 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:47:20,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:47:20,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:47:20,671 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:47:21,404 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:21,853 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:47:26,854 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:30,622 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:47:32,489 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:35,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:47:35,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:47:35,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:47:38,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:42,703 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:47:44,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:49,391 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:47:50,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:47:50,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:47:50,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:47:50,758 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:47:51,853 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:47:54,854 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:00,757 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:00,835 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:48:05,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:48:05,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:48:05,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:48:05,908 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:10,906 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:48:11,318 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:16,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:20,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:48:20,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:48:20,673 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:48:20,980 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:48:21,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:21,854 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:48:26,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:31,063 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:48:32,068 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:35,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:48:35,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:48:35,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:48:37,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:42,692 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:43,250 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:48:47,694 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:50,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:48:50,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:48:51,854 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:48:52,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:48:53,350 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:48:57,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:03,460 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:49:03,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:05,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:49:05,679 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:49:09,407 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:11,517 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:49:15,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:20,588 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:49:20,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:49:20,673 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:49:20,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:21,855 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:49:25,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:30,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:32,663 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:49:35,672 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:49:35,673 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:49:35,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:41,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:42,744 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:49:46,821 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:50,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:49:50,677 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:49:51,855 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:49:51,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:49:52,818 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:49:56,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:00,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:50:00,517 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:50:00,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:50:00,518 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:50:00,881 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:50:01,885 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:02,519 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:02,890 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:05,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:50:05,677 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:50:08,407 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:10,941 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:14,151 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:19,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:20,677 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:50:20,678 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:50:21,029 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:21,856 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:50:24,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:29,858 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:35,128 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:35,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:50:35,677 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:50:35,858 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:41,408 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:45,196 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:46,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:51,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:50:51,243 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:50:51,244 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:50:51,857 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:50:51,860 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:50:53,271 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:50:57,451 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:02,974 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:03,348 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:51:06,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:51:06,243 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:51:06,243 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:51:08,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:13,427 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:51:14,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:19,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:21,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:51:21,243 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:51:21,243 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:51:21,858 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:51:23,499 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:51:24,858 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:29,867 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:35,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:35,634 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:51:36,242 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:51:36,243 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:51:36,243 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:51:40,466 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:45,733 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:51:46,122 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:51,124 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:51:51,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:51:51,677 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:51:51,678 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:51:51,936 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:51:54,811 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:51:56,860 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:01,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:02,870 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:52:06,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:52:06,676 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:52:07,835 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:12,937 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:52:13,467 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:18,469 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:21,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:52:21,676 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:52:21,887 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:52:23,010 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:52:23,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:28,889 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:33,080 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:52:34,428 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:36,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:52:36,677 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:52:39,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:45,171 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:52:45,814 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:50,816 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:52:51,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:52:51,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:52:51,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:52:51,892 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:52:55,235 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:52:56,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:01,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:06,148 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:06,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:53:06,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:53:06,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:53:09,333 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:53:11,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:16,548 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:19,417 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:53:21,677 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:53:21,678 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:53:21,887 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:53:21,887 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:26,889 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:27,264 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:53:27,265 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:53:27,266 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:53:27,268 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:53:27,479 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:53:29,493 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:53:32,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:36,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:53:36,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:53:36,679 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:53:37,893 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:39,612 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:53:42,894 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:48,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:49,707 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:53:51,677 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:53:51,678 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:53:51,873 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:53:53,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:59,268 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:53:59,776 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:54:04,269 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:06,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:54:06,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:54:06,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:54:09,847 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:54:09,968 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:14,970 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:17,916 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:54:20,355 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:21,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:54:21,678 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:54:21,886 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:54:25,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:28,988 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:54:31,283 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:36,285 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:36,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:54:36,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:54:39,054 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:54:41,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:46,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:49,132 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:54:51,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:51,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:54:51,679 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:54:51,928 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:54:56,930 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:54:59,194 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:55:02,467 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:06,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:55:06,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:55:08,305 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:09,274 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:55:13,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:18,307 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:19,346 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:55:21,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:55:21,679 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:55:21,875 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:55:23,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:29,177 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:29,420 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:55:34,178 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:36,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:55:36,683 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:55:36,684 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:55:39,501 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:55:39,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:44,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:49,579 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:55:50,715 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:51,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:55:51,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:55:51,867 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:55:55,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:55:59,647 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:56:01,416 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:06,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:06,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:56:06,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:56:07,710 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:56:11,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:16,928 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:19,779 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:56:21,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:56:21,680 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:56:21,938 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:56:21,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:26,940 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:29,855 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:56:32,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:36,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:56:36,682 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:56:38,674 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:39,925 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:56:43,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:47,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 17:56:47,580 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 17:56:47,580 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 17:56:47,582 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 17:56:47,985 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 17:56:49,584 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:56:49,992 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:56:51,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:56:51,681 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:56:51,865 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:56:54,866 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:00,063 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:00,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:05,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:06,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:57:06,681 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:57:08,120 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:10,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:15,924 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:20,195 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:21,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:21,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:57:21,686 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:57:21,866 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:57:26,833 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:27,252 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:31,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:36,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:57:36,682 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:57:36,902 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:37,399 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:41,903 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:43,427 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:44,433 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:46,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:51,473 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:51,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:57:51,682 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:57:51,890 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:57:52,478 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:57:52,890 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:57:57,892 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:01,538 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:58:03,812 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:08,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:58:08,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:58:08,177 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:58:09,373 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:11,614 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:58:14,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:19,688 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:58:19,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:21,867 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:58:23,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:58:23,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:58:23,175 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:58:25,433 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:29,757 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:58:31,027 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:36,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:39,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:58:39,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:58:39,451 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:58:41,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:41,838 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:58:46,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:51,867 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:58:51,869 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:58:53,916 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:58:54,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:58:54,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:58:54,449 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:58:57,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:03,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:03,978 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:59:08,164 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:09,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:59:09,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:59:09,449 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:59:13,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:16,058 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:59:18,376 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:21,869 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:59:23,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:24,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:59:24,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:59:24,450 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:59:26,127 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:59:29,620 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:35,119 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:36,205 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:59:39,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:59:39,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:59:39,450 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:59:40,614 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:46,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:46,281 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:59:51,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 17:59:51,870 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 17:59:54,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 17:59:54,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 17:59:54,452 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 17:59:56,349 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 17:59:56,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:01,687 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:04,458 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:00:04,460 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:00:04,460 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:00:04,463 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:00:05,421 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:00:06,428 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:00:07,464 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:09,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:00:09,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:00:09,450 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:00:12,684 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:15,493 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:00:17,960 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:21,873 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:00:23,876 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:24,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:00:24,451 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:00:24,453 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:00:27,572 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:00:29,435 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:34,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:37,649 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:00:39,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:00:39,450 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:39,450 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:00:39,451 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:00:44,647 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:47,727 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:00:49,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:51,872 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:00:54,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:00:54,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:00:54,912 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:00:54,914 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:00:57,800 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:00,301 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:05,302 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:07,874 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:09,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:01:09,911 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:01:11,158 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:16,159 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:17,951 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:21,884 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:01:21,885 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:26,139 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:01:26,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:01:26,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:01:27,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:30,030 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:32,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:37,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:40,106 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:41,139 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:01:41,140 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:01:43,314 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:48,487 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:50,173 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:51,873 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:01:53,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:01:56,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:01:56,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:01:56,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:01:58,235 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:01:59,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:04,274 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:09,671 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:10,312 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:02:11,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:02:11,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:02:15,331 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:18,394 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:02:21,291 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:21,874 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:02:26,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:02:26,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:02:26,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:02:26,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:28,470 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:02:32,289 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:37,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:38,538 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:02:41,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:02:41,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:02:43,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:48,453 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:48,614 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:02:51,874 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:02:53,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:02:56,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:02:56,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:02:57,675 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:02:59,387 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:04,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:07,749 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:09,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:11,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:03:11,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:03:15,401 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:15,809 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:20,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:21,874 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:03:25,684 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:25,685 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:03:25,687 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:03:25,687 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:03:25,690 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:03:25,886 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:03:26,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:03:26,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:03:26,891 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:27,897 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:31,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:37,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:37,972 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:41,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:03:41,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:03:42,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:47,696 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:48,058 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:51,875 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:03:52,876 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:03:56,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:03:56,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:03:58,129 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:03:58,848 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:03,850 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:08,196 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:04:09,379 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:11,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:04:11,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:04:15,309 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:18,272 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:04:21,084 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:21,876 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:04:26,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:04:26,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:04:26,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:28,500 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:04:31,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:36,557 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:04:36,626 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:41,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:04:41,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:04:42,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:46,659 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:04:48,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:51,878 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:04:53,880 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:04:55,871 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:04:56,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:04:56,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:04:59,349 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:04,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:05,941 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:05:09,920 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:11,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:05:11,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:05:15,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:16,016 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:05:20,759 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:21,879 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:05:26,107 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:05:26,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:05:26,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:05:26,297 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:31,298 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:36,178 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:05:37,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:41,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:05:41,142 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:05:42,347 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:46,248 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:05:48,029 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:51,880 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:05:53,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:05:56,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:05:56,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:05:56,325 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:05:59,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:04,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:09,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:11,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:06:11,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:06:15,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:18,614 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:06:20,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:21,881 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:06:25,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:26,143 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:06:26,144 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:06:28,689 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:06:31,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:36,530 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:40,810 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:06:41,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:06:41,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:06:42,335 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:47,336 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:50,889 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:06:51,881 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:06:52,882 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:57,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:06:57,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:06:57,028 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:06:58,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:06:58,205 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:06:58,206 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:06:58,207 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:06:58,209 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:06:58,942 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:07:00,951 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:07:03,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:08,986 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:11,053 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:07:12,024 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:07:12,025 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:07:14,197 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:19,989 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:20,127 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:07:21,882 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:07:25,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:27,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:07:27,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:07:30,196 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:07:31,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:36,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:40,264 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:07:41,985 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:42,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:07:42,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:07:47,224 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:50,348 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:07:51,882 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:07:52,883 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:07:57,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:07:57,025 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:07:58,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:03,827 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:04,452 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:08:08,841 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:12,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:08:12,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:08:14,287 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:14,520 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:08:19,288 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:21,894 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:08:24,595 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:08:25,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:27,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:08:27,025 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:08:30,247 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:34,666 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:08:36,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:41,021 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:42,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:08:42,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:08:42,729 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:08:46,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:51,222 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:51,884 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:08:52,803 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:08:56,885 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:08:57,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:08:57,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:09:02,281 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:02,879 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:09:07,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:12,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:09:12,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:09:12,302 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:12,954 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:09:17,303 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:21,884 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:09:22,885 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:23,027 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:09:27,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:09:27,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:09:28,299 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:33,104 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:09:33,544 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:38,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:42,025 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:09:42,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:09:43,189 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:09:44,263 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:49,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:51,885 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:09:53,256 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:09:54,886 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:09:57,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:09:57,026 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:10:00,282 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:02,326 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:05,496 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:10,497 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:12,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:10:12,027 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:10:12,449 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:16,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:20,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:10:20,797 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:10:20,797 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:10:20,799 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:10:21,522 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:10:21,800 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:21,885 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:10:22,529 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:23,534 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:26,887 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:27,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:10:27,027 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:10:32,597 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:32,620 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:37,622 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:42,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:10:42,027 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:10:42,671 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:43,264 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:48,265 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:51,886 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:10:52,756 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:10:53,895 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:10:57,026 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:10:57,027 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:10:59,271 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:02,827 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:11:04,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:09,574 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:12,027 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:11:12,028 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:11:12,893 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:11:15,292 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:21,064 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:21,886 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:11:22,970 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:11:26,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:27,028 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:11:27,030 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:11:27,031 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:11:31,952 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:33,046 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:11:36,954 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:41,115 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:11:42,027 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:42,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:11:42,046 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:11:47,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:51,189 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:11:51,887 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:11:52,888 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:11:57,860 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:11:57,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:11:57,863 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:11:58,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:01,268 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:12:03,987 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:09,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:11,331 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:12:12,860 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:12:12,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:12:12,862 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:12:15,066 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:20,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:21,424 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:12:21,887 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:12:25,889 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:27,860 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:12:27,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:12:27,862 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:12:30,491 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:12:31,055 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:36,056 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:40,699 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:12:41,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:42,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:12:42,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:12:42,864 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:12:47,055 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:50,762 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:12:51,899 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:12:52,900 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:57,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:12:57,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:12:57,862 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:12:58,040 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:12:58,832 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:13:03,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:08,732 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:08,903 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:13:12,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:13:12,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:13:12,862 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:13:14,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:18,973 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:13:19,252 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:21,907 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:13:24,928 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:27,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:13:27,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:13:27,862 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:13:29,151 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:13:30,016 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:35,018 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:37,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:13:37,131 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:13:37,132 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:13:37,134 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:13:37,221 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:13:39,233 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:13:40,145 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:42,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:13:42,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:13:42,863 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:13:46,113 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:49,306 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:13:51,163 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:51,907 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:13:56,909 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:13:57,861 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:13:57,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:13:57,863 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:13:59,375 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:02,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:07,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:09,446 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:12,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:12,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:14:12,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:14:12,863 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:14:18,554 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:19,533 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:21,908 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:14:23,928 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:27,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:14:27,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:14:27,863 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:14:29,084 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:29,619 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:34,086 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:38,702 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:39,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:42,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:14:42,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:14:42,867 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:14:45,148 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:46,759 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:51,022 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:51,909 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:14:56,843 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:14:56,960 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:14:57,862 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:14:57,863 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:14:57,863 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:15:02,093 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:06,919 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:15:07,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:12,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:14,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:15:14,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:15:14,193 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:15:16,985 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:15:18,427 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:21,909 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:15:23,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:27,053 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:15:29,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:15:29,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:15:29,192 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:15:29,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:34,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:37,138 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:15:40,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:44,803 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:15:44,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:15:44,805 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:15:45,976 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:47,214 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:15:51,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:51,910 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:15:56,872 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:15:57,278 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:15:59,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:15:59,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:15:59,808 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:16:02,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:07,086 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:07,348 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:12,087 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:14,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:16:14,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:16:14,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:16:17,422 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:17,944 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:21,910 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:16:23,911 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:27,495 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:29,044 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:29,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:16:29,806 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:16:34,057 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:37,561 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:39,993 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:44,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:16:44,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:16:45,016 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:47,818 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:50,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:51,911 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:16:55,572 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:16:55,574 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:16:55,575 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:16:55,577 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:16:55,875 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:16:56,578 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:16:56,880 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:57,884 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:16:59,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:16:59,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:17:02,068 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:06,949 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:17:07,494 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:12,496 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:14,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:17:14,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:17:17,025 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:17:18,097 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:21,912 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:17:23,915 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:25,078 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:17:29,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:29,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:17:29,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:17:29,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:17:35,020 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:35,156 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:17:40,021 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:44,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:17:44,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:17:44,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:17:45,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:45,247 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:17:50,065 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:51,914 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:17:55,320 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:17:55,724 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:17:59,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:17:59,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:17:59,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:18:01,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:05,412 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:18:06,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:11,373 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:14,806 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:18:14,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:18:14,807 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:18:15,496 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:18:17,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:21,915 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:18:22,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:25,637 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:18:27,592 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:29,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:18:29,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:18:29,811 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:18:33,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:35,720 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:18:38,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:44,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:18:44,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:18:44,808 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:18:45,046 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:45,852 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:18:50,046 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:51,916 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:18:55,238 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:18:55,916 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:18:59,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:18:59,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:18:59,811 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:19:00,983 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:05,990 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:19:06,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:11,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:13,042 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:19:14,807 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:19:14,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:19:14,808 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:19:17,059 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:21,917 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:19:22,308 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:23,143 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:19:27,309 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:31,208 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:19:31,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:19:31,211 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:19:32,397 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:33,219 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:19:37,398 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:42,813 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:43,289 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:19:46,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:19:46,209 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:19:48,468 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:51,917 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:19:53,359 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:19:54,183 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:19:59,184 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:01,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:20:01,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:20:01,212 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:20:03,428 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:20:04,921 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:09,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:11,935 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:20:11,936 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:20:11,937 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:20:11,939 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:20:12,494 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:20:13,499 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:20:14,941 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:16,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:20:16,209 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:20:20,436 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:21,918 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:20:23,584 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:20:25,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:30,921 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:31,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:20:31,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:20:31,212 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:20:33,659 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:20:36,869 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:42,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:43,737 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:20:46,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:20:46,210 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:20:48,431 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:51,919 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:20:53,821 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:20:54,205 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:20:59,206 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:01,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:21:01,212 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:21:01,213 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:21:03,901 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:21:04,961 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:09,963 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:14,000 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:21:15,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:16,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:21:16,211 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:21:20,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:21,919 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:21:24,073 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:21:25,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:30,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:31,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:21:31,212 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:21:34,173 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:21:35,983 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:40,985 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:43,243 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:21:46,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:21:46,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:46,212 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:21:51,442 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:21:51,930 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:21:53,335 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:21:56,932 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:01,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:22:01,212 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:22:01,943 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:03,411 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:22:06,945 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:12,890 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:13,476 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:22:16,210 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:22:16,211 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:22:18,510 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:21,923 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:22:23,552 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:22:23,924 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:28,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:31,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:22:31,211 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:22:34,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:37,642 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:22:40,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:45,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:46,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:22:46,211 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:22:47,716 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:22:50,425 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:51,923 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:22:55,925 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:22:57,791 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:01,211 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:23:01,212 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:01,212 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:23:06,327 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:07,862 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:11,327 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:16,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:23:16,214 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:23:16,381 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:17,939 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:21,382 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:21,924 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:23:26,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:28,015 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:31,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:23:31,214 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:23:32,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:35,837 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:23:35,840 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:23:35,840 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:23:35,842 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:23:36,077 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:23:37,845 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:38,086 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:42,847 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:46,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:23:46,214 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:23:48,365 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:48,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:51,924 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:23:53,926 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:23:57,435 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:23:59,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:01,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:24:01,214 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:24:05,351 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:07,511 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:24:10,352 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:15,570 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:24:15,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:16,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:24:16,215 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:24:21,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:21,926 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:24:25,640 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:24:27,199 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:31,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:24:31,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:24:31,217 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:24:32,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:35,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:24:38,251 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:43,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:45,850 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:24:46,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:24:46,215 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:24:48,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:51,927 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:24:53,930 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:24:55,941 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:24:59,333 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:01,216 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:25:01,218 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:25:01,219 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:25:04,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:06,021 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:25:09,939 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:15,626 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:16,100 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:25:16,215 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:25:16,215 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:25:21,466 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:21,930 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:25:26,170 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:25:27,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:32,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:25:32,347 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:25:32,349 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:25:32,499 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:36,237 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:25:38,443 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:43,988 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:44,293 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:25:47,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:25:47,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:25:47,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:25:49,579 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:51,929 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:25:54,930 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:25:59,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:02,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:26:02,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:26:05,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:10,586 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:15,587 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:17,346 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:26:17,346 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:26:21,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:21,933 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:26:25,623 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:26:26,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:31,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:33,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:26:33,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:26:33,534 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:26:35,702 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:26:37,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:42,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:45,780 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:26:47,923 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:48,531 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:26:48,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:26:48,532 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:26:51,932 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:26:52,936 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:26:55,850 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:26:58,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:03,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:27:03,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:27:03,536 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:27:03,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:05,912 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:27:09,162 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:14,959 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:15,977 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:27:18,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:27:18,532 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:27:20,727 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:21,933 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:27:24,047 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:27:24,048 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:27:24,049 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:27:24,052 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:27:25,036 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:27:26,042 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:27:26,053 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:31,054 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:33,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:27:33,536 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:27:36,124 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:27:36,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:41,585 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:46,233 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:27:46,934 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:48,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:27:48,533 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:27:51,935 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:27:51,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:27:56,298 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:27:57,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:02,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:03,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:28:03,534 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:28:06,369 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:28:07,831 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:12,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:16,444 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:28:18,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:28:18,533 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:28:18,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:21,937 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:28:24,074 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:24,520 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:28:29,075 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:33,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:28:33,533 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:28:34,592 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:28:34,782 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:39,783 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:44,667 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:28:45,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:48,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:28:48,533 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:28:51,701 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:28:51,937 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:28:54,738 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:28:57,437 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:03,331 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:03,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:29:03,534 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:29:04,805 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:29:08,805 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:13,880 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:29:13,972 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:18,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:29:18,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:29:18,536 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:29:19,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:21,939 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:29:23,980 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:29:24,940 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:29,941 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:33,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:29:33,534 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:29:34,056 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:29:35,703 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:40,705 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:44,133 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:29:46,227 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:48,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:29:48,534 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:29:51,739 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:29:51,939 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:29:54,205 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:29:57,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:02,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:03,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:30:03,534 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:30:04,266 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:30:08,695 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:13,849 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:14,332 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:30:18,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:30:18,536 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:30:18,537 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:30:19,736 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:21,939 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:30:24,413 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:30:24,940 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:29,948 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:33,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:30:33,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:30:34,508 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:30:35,732 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:40,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:42,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:30:42,092 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:30:42,093 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:30:42,096 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:30:42,565 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:30:44,576 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:30:46,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:48,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:30:48,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:30:51,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:30:51,940 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:30:54,652 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:30:57,004 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:02,872 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:03,534 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:31:03,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:31:04,724 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:31:08,725 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:14,667 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:14,804 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:31:18,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:31:18,535 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:31:19,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:21,940 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:31:24,941 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:29,942 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:33,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:31:33,536 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:31:35,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:40,826 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:45,826 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:48,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:31:48,536 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:31:51,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:31:51,943 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:31:54,025 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:31:57,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:02,097 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:03,537 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:32:03,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:32:03,540 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:32:06,107 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:32:07,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:12,561 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:16,182 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:32:18,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:18,537 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:32:18,537 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:32:21,943 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:32:23,945 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:26,258 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:32:29,660 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:33,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:32:33,539 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:32:35,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:36,340 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:32:40,551 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:45,724 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:46,412 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:32:48,537 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:32:48,537 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:32:50,837 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:51,945 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:32:56,218 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:32:56,483 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:33:01,221 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:04,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:33:04,043 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:33:04,044 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:33:06,252 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:06,590 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:33:11,253 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:16,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:16,664 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:33:19,043 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:33:19,043 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:33:19,044 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:33:21,334 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:21,947 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:33:26,748 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:33:27,112 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:32,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:34,055 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:33:34,056 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:33:34,057 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:33:36,897 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:33:37,223 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:42,224 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:47,072 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:33:47,810 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:49,054 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:33:49,054 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:33:51,946 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:33:52,978 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:58,877 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:33:59,150 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:03,879 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:04,201 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:34:04,201 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:34:04,203 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:34:06,201 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:09,532 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:14,533 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:16,281 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:19,201 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:34:19,201 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:34:20,455 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:21,947 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:34:25,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:26,369 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:30,810 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:34,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:34:34,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:34:34,370 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:34:35,519 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:34:35,520 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:34:35,521 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:34:35,524 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:34:36,452 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:34:36,454 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:36,526 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:41,527 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:46,520 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:47,294 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:49,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:34:49,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:34:49,368 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:34:51,949 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:34:52,952 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:34:56,588 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:34:58,029 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:03,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:04,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:35:04,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:35:04,369 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:35:06,670 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:35:08,849 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:13,851 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:16,752 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:35:19,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:19,367 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:35:19,368 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:35:21,950 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:35:24,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:26,826 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:35:29,959 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:34,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:35:34,368 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:35:34,893 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:35:35,615 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:40,619 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:44,965 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:35:46,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:49,368 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:35:49,368 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:35:51,574 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:35:51,951 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:35:55,041 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:35:56,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:01,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:04,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:36:04,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:36:04,370 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:36:05,115 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:36:07,529 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:12,530 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:15,187 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:36:17,707 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:19,370 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:36:19,374 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:36:19,376 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:36:21,961 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:36:23,285 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:25,262 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:36:28,286 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:33,674 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:34,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:36:34,371 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:36:35,331 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:36:39,567 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:44,424 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:36:44,748 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:49,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:36:49,373 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:36:50,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:36:51,954 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:36:54,485 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:36:55,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:00,682 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:02,543 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:37:04,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:37:04,372 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:37:06,550 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:11,552 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:14,666 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:37:17,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:20,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:37:20,966 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:37:20,967 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:37:21,954 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:37:22,722 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:37:23,069 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:28,070 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:32,802 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:37:33,717 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:35,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:37:35,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:37:39,203 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:42,906 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:37:44,696 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:49,698 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:50,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:37:50,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:37:51,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:37:51,447 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:37:51,447 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:37:51,449 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:37:51,954 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:37:51,971 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:37:52,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:37:54,955 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:37:59,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:03,050 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:38:05,770 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:05,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:38:05,965 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:38:11,229 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:16,230 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:19,160 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:38:20,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:38:20,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:38:21,955 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:38:21,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:27,073 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:27,227 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:38:32,086 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:35,965 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:38:35,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:38:37,204 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:37,307 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:38:42,205 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:47,347 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:49,434 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:38:50,966 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:38:50,966 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:38:51,955 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:38:52,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:58,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:38:59,504 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:39:03,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:05,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:39:05,971 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:39:08,566 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:08,608 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:39:13,567 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:16,665 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:39:19,327 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:20,968 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:39:20,969 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:39:21,956 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:39:24,957 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:26,747 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:39:29,982 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:34,984 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:35,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:39:35,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:39:35,969 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:39:36,821 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:39:40,172 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:45,182 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:46,905 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:39:50,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:50,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:39:50,969 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:39:51,956 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:39:55,874 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:39:56,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:00,875 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:05,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:40:05,969 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:40:06,231 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:07,061 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:11,232 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:15,123 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:16,323 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:21,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:21,958 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:40:22,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:40:22,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:40:22,194 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:40:25,191 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:27,160 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:32,162 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:35,262 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:37,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:40:37,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:40:37,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:37,193 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:40:42,461 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:45,340 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:47,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:51,960 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:40:52,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:40:52,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:40:52,196 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:40:53,310 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:40:55,417 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:40:58,310 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:04,075 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:05,476 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:41:07,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:41:07,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:41:07,192 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:41:09,519 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:12,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:41:12,186 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:41:12,187 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:41:12,191 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:41:12,533 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:41:13,537 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:41:15,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:20,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:21,960 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:41:22,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:41:22,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:41:22,193 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:41:23,604 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:41:25,399 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:31,358 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:31,670 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:41:36,359 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:37,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:41:37,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:41:37,196 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:41:41,452 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:41,748 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:41:46,453 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:50,813 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:41:51,750 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:41:51,960 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:41:52,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:41:52,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:41:52,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:41:57,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:00,891 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:02,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:07,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:42:07,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:42:07,259 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:42:08,701 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:08,971 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:13,702 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:19,039 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:19,678 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:21,961 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:42:22,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:42:22,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:42:22,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:42:25,535 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:27,091 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:30,538 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:35,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:37,157 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:37,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:42:37,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:42:37,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:42:41,460 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:46,644 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:47,235 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:51,645 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:42:51,961 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:42:52,257 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:42:52,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:42:52,258 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:42:57,317 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:42:57,440 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:02,443 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:07,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:43:07,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:43:07,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:43:07,437 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:43:07,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:12,473 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:17,512 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:43:17,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:21,962 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:43:22,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:43:22,260 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:43:22,261 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:43:23,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:27,597 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:43:29,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:34,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:37,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:43:37,258 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:43:37,259 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:43:37,655 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:43:39,435 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:45,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:45,718 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:43:50,342 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:51,964 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:43:52,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:43:52,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:43:52,477 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:43:55,470 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:43:55,790 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:00,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:05,864 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:06,034 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:07,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:44:07,475 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:44:11,740 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:13,958 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:17,560 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:21,965 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:44:22,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:44:22,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:44:22,477 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:44:22,698 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:22,816 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:44:22,817 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:44:22,818 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:44:22,820 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:44:23,021 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:44:25,031 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:26,035 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:27,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:33,048 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:33,118 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:37,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:44:37,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:44:38,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:43,182 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:44,363 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:49,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:51,965 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:44:52,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:44:52,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:44:53,266 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:44:54,651 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:44:59,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:03,339 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:45:05,007 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:07,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:45:07,477 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:45:10,739 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:11,413 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:45:15,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:20,771 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:21,966 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:45:22,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:45:22,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:45:23,499 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:45:26,701 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:31,704 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:33,634 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:45:36,705 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:37,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:45:37,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:45:42,546 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:43,713 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:45:47,547 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:51,770 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:45:51,966 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:45:52,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:45:52,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:45:52,732 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:45:57,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:01,850 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:46:03,363 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:07,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:46:07,477 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:46:08,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:13,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:18,721 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:21,968 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:46:22,476 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:46:22,477 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:46:23,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:29,073 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:30,028 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:46:34,074 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:37,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:46:37,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:46:37,479 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:46:39,690 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:40,101 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:46:44,692 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:50,166 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:46:50,496 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:46:51,968 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:46:52,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:46:52,479 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:46:55,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:00,253 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:01,076 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:06,078 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:07,320 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:07,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:47:07,478 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:47:11,810 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:16,812 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:17,396 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:21,969 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:21,969 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:47:22,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:47:22,478 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:47:27,174 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:29,508 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:32,175 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:37,478 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:47:37,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:37,479 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:47:37,566 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:42,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:47,633 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:47,737 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:51,969 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:47:52,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:47:52,481 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:47:52,481 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:47:53,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:47:55,625 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:47:55,626 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:47:55,626 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:47:55,629 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:47:55,705 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:47:57,714 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:47:59,630 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:04,632 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:05,770 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:48:07,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:48:07,479 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:48:09,693 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:15,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:15,835 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:48:20,214 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:21,971 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:48:23,109 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:48:23,110 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:48:23,111 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:48:25,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:25,967 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:48:30,256 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:35,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:36,044 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:48:38,109 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:48:38,110 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:48:38,110 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:48:41,362 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:44,109 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:48:47,028 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:51,971 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:48:52,716 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:48:53,109 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:48:53,110 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:48:53,110 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:48:54,178 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:48:58,338 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:04,085 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:04,250 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:49:08,109 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:49:08,110 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:49:09,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:14,312 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:49:14,612 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:19,614 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:21,972 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:49:22,370 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:49:23,110 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:49:23,110 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:49:25,329 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:30,777 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:32,437 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:49:35,778 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:38,669 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:49:38,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:49:38,672 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:49:40,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:42,512 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:49:45,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:51,366 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:49:51,582 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:49:51,975 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:49:53,669 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:49:53,670 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:49:53,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:49:56,876 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:01,652 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:50:02,319 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:07,320 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:09,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:50:09,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:50:09,567 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:50:11,716 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:50:12,639 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:17,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:21,782 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:50:21,973 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:50:22,974 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:24,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:50:24,564 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:50:24,564 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:50:28,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:29,851 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:50:33,656 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:39,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:50:39,564 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:50:39,564 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:50:39,800 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:39,928 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:50:44,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:48,006 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:50:50,637 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:51,974 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:50:54,869 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:50:54,871 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:50:54,873 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:50:55,813 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:50:58,072 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:00,814 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:05,312 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:51:05,315 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:51:05,316 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:51:05,319 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:51:06,121 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:51:06,122 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:06,321 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:08,132 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:09,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:51:09,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:51:09,871 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:51:12,084 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:16,186 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:17,627 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:21,979 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:51:22,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:24,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:51:24,870 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:51:24,871 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:51:26,282 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:28,059 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:33,061 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:34,351 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:38,213 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:41,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:51:41,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:51:41,097 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:51:44,123 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:44,425 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:49,124 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:51,976 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:51:52,517 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:51:54,977 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:51:56,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:51:56,096 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:52:00,388 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:02,592 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:52:05,856 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:10,857 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:11,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:52:11,096 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:52:11,098 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:52:12,678 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:52:16,393 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:21,767 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:52:21,871 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:21,976 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:52:26,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:52:26,096 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:52:27,397 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:31,913 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:52:32,867 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:37,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:41,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:52:41,096 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:52:43,371 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:48,119 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:52:48,666 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:51,977 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:52:53,978 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:52:56,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:52:56,096 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:52:58,194 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:52:59,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:04,297 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:08,267 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:53:09,931 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:11,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:53:11,097 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:53:15,339 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:18,339 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:53:20,797 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:21,977 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:53:26,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:53:26,097 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:53:26,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:26,411 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:53:31,262 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:36,489 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:53:36,956 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:41,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:53:41,097 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:53:42,286 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:46,558 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:53:47,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:51,979 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:53:52,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:53:56,097 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:53:56,102 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:53:56,617 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:53:58,344 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:03,345 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:04,798 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:54:08,375 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:12,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:54:12,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:54:12,014 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:54:14,156 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:14,870 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:54:19,157 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:21,980 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:54:22,502 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:54:22,504 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:54:22,505 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:54:22,507 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:54:22,929 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:54:23,936 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:54:24,509 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:27,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:54:27,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:54:27,012 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:54:30,209 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:34,012 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:54:35,543 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:40,545 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:42,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:54:42,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:54:42,012 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:54:44,075 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:54:46,182 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:51,840 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:51,980 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:54:52,138 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:54:56,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:54:57,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:54:57,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:54:57,012 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:55:02,330 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:55:02,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:07,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:12,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:55:12,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:55:12,012 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:55:12,407 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:55:13,268 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:18,270 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:21,981 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:55:22,589 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:55:23,981 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:28,189 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:55:28,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:55:28,193 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:55:29,136 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:30,652 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:55:34,136 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:39,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:40,753 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:55:43,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:55:43,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:55:43,191 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:55:45,398 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:48,806 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:55:50,631 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:51,981 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:55:55,984 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:55:58,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:55:58,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:55:58,191 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:55:58,936 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:01,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:06,392 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:09,004 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:12,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:13,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:56:13,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:56:13,191 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:56:17,498 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:18,064 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:21,981 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:56:22,983 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:26,136 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:28,190 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:56:28,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:56:28,191 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:56:28,394 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:33,396 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:36,207 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:39,007 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:43,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:56:43,193 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:56:43,196 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:56:44,918 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:46,273 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:49,919 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:51,983 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:56:54,331 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:56:55,410 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:56:58,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:56:58,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:56:58,192 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:57:00,419 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:04,394 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:05,718 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:10,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:13,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:57:13,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:57:13,192 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:57:14,500 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:16,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:21,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:21,983 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:57:24,577 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:27,173 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:28,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:57:28,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:57:28,192 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:57:31,753 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 18:57:31,756 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 18:57:31,757 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 18:57:31,759 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 18:57:32,643 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 18:57:32,644 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:32,761 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:34,650 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:37,764 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:42,700 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:42,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:43,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:57:43,259 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:57:43,260 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:57:48,438 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:51,983 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:57:52,770 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:57:53,984 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:58,987 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:57:59,115 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:57:59,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:57:59,119 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:58:02,887 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:58:04,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:09,185 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:12,953 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:58:14,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:58:14,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:58:14,324 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:19,326 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:21,023 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:58:21,984 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:58:24,985 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:29,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:58:29,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:58:30,363 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:31,096 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:58:35,364 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:40,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:41,166 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:58:44,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:58:44,116 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:58:46,365 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:50,307 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:58:51,949 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:51,985 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:58:56,986 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:58:59,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:58:59,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:59:00,384 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:02,393 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:02,485 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:07,487 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:10,452 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:12,564 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:14,116 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:59:14,117 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:59:17,953 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:18,511 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:21,985 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:59:22,987 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:28,244 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:28,587 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:29,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:59:29,118 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:59:33,296 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:38,430 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:38,660 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:43,431 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:45,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 18:59:45,142 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 18:59:45,143 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 18:59:46,714 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 18:59:49,169 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:51,986 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 18:59:54,987 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 18:59:56,774 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:00,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:00:00,141 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:00,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:00:05,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:06,837 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:10,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:14,895 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:15,140 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:00:15,141 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:00:16,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:21,369 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:21,987 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:00:24,960 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:26,815 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:31,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:00:31,659 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:00:31,660 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:00:31,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:35,027 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:37,764 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:41,947 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:00:41,949 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:00:41,950 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:00:41,952 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:00:42,084 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:00:42,954 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:43,091 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:44,097 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:46,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:00:46,659 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:00:46,659 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:00:48,820 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:51,987 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:00:53,160 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:00:53,988 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:00:58,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:01,214 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:01,658 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:01:01,659 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:01:01,660 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:01:04,849 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:10,194 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:11,275 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:15,195 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:17,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:01:17,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:01:17,154 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:01:19,334 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:20,201 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:21,988 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:01:25,989 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:29,395 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:30,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:32,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:01:32,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:01:32,153 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:01:36,405 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:39,454 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:42,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:47,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:01:47,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:47,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:01:47,153 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:01:48,527 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:51,988 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:01:52,990 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:01:56,584 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:01:58,978 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:02,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:02:02,152 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:02:02,153 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:02:04,382 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:09,382 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:14,384 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:16,694 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:02:17,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:02:17,153 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:02:17,154 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:02:19,385 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:21,990 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:02:24,605 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:26,766 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:02:29,606 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:32,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:02:32,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:02:32,450 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:02:34,820 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:02:35,501 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:40,502 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:44,926 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:02:45,521 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:47,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:02:47,449 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:02:47,449 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:02:50,673 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:02:51,991 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:02:55,050 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:02:56,472 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:01,474 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:02,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:03:02,448 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:03:05,138 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:03:06,922 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:11,924 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:15,211 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:03:16,938 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:17,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:03:17,448 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:03:21,993 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:03:21,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:25,344 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:03:27,884 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:32,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:03:32,448 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:03:33,409 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:03:33,670 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:38,674 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:43,484 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:03:44,074 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:47,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:03:47,448 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:03:49,654 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:51,993 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:03:53,551 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:03:54,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:03:59,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:00,487 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:04:00,488 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:04:00,489 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:04:00,491 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:04:00,607 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:04:01,612 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:02,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:04:02,449 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:04:02,618 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:05,657 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:10,696 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:11,447 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:16,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:18,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:04:18,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:04:18,403 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:04:20,759 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:21,498 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:21,993 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:04:26,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:30,825 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:32,401 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:33,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:04:33,401 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:04:33,401 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:04:37,638 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:40,909 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:42,642 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:47,643 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:48,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:04:48,400 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:04:48,976 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:04:51,994 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:04:52,994 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:58,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:04:59,048 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:05:03,036 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:03,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:05:03,401 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:05:08,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:09,121 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:05:13,096 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:18,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:18,400 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:05:18,401 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:05:19,203 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:05:21,994 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:05:23,995 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:29,255 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:29,278 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:05:33,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:05:33,404 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:05:34,599 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:37,332 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:05:39,629 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:44,631 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:47,391 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:05:48,401 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:05:48,401 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:05:49,648 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:51,996 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:05:54,999 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:05:57,467 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:00,102 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:03,510 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:06:03,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:06:03,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:06:05,517 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:05,543 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:10,544 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:15,582 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:15,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:18,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:06:18,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:06:18,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:06:20,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:21,997 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:06:25,662 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:26,389 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:31,390 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:33,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:06:33,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:06:33,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:06:35,736 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:36,822 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:41,825 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:45,811 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:47,439 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:48,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:06:48,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:06:48,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:06:51,999 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:06:53,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:06:55,880 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:06:58,027 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:03,028 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:03,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:07:03,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:07:03,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:07:04,955 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:07:08,735 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:13,736 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:17,785 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:07:17,786 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:07:17,787 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:07:17,789 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:07:18,037 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:07:18,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:07:18,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:07:18,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:07:18,884 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:19,045 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:07:21,055 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:07:21,998 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:07:23,999 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:29,000 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:33,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:07:33,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:07:33,513 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:07:34,683 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:39,684 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:44,696 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:48,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:07:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:07:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:07:49,775 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:07:52,001 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:07:55,002 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:00,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:03,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:08:03,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:08:03,566 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:08:05,800 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:10,801 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:15,802 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:18,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:08:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:08:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:08:21,737 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:21,999 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:08:27,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:32,034 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:33,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:08:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:08:33,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:08:37,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:42,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:47,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:48,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:08:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:08:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:08:52,000 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:08:53,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:08:58,001 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:03,002 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:03,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:09:03,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:09:03,566 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:09:08,808 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:13,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:18,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:09:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:09:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:09:19,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:22,001 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:09:25,002 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:30,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:33,512 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:09:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:09:33,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:09:35,736 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:40,737 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:45,738 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:48,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:09:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:09:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:09:50,795 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:09:52,001 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:09:56,002 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:01,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:03,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:10:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:10:03,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:10:06,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:11,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:16,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:18,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:10:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:10:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:10:21,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:22,001 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:10:27,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:32,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:33,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:10:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:10:33,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:10:37,798 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:42,799 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:47,800 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:48,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:10:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:10:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:10:52,002 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:10:53,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:10:58,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:03,004 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:03,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:11:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:11:03,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:11:08,769 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:13,770 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:18,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:11:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:11:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:11:18,804 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:22,002 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:11:24,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:29,004 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:33,513 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:11:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:11:33,557 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:11:34,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:39,760 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:44,761 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:48,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:11:48,556 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:11:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:11:49,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:11:52,013 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:11:55,015 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:00,016 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:03,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:12:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:12:03,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:12:05,730 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:10,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:15,731 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:18,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:12:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:12:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:12:21,714 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:22,003 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:12:27,004 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:32,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:33,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:12:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:12:33,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:12:37,742 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:42,743 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:47,744 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:48,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:12:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:12:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:12:52,003 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:12:53,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:12:58,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:03,006 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:03,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:13:03,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:13:03,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:13:08,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:13,780 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:18,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:13:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:13:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:13:19,777 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:22,004 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:13:25,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:30,006 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:33,514 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:13:33,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:13:33,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:13:35,761 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:40,762 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:45,763 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:48,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:13:48,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:13:48,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:13:50,767 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:13:52,004 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:13:56,005 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:01,006 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:03,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:14:03,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:14:03,566 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:14:06,740 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:11,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:16,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:18,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:14:18,555 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:14:18,556 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:14:21,811 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:22,007 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:14:27,008 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:32,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:33,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:14:33,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:14:33,576 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:14:37,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:42,809 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:47,810 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:48,515 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:14:48,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:14:48,566 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:14:52,007 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:14:53,008 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:14:58,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:03,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:03,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:15:03,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:15:03,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:15:08,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:13,773 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:18,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:15:18,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:15:18,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:15:19,695 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:22,008 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:15:25,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:30,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:33,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:15:33,565 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:15:33,566 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:15:35,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:40,794 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:45,795 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:48,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:15:48,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:15:48,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:15:51,705 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:15:52,008 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:15:57,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:02,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:03,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:16:03,575 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:16:03,576 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:16:07,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:12,741 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:17,742 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:18,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:16:18,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:16:18,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:16:22,008 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:16:23,009 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:28,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:33,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:33,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:16:33,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:16:33,517 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:16:38,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:43,724 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:48,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:16:48,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:16:48,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:16:48,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:52,009 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:16:54,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:16:59,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:03,516 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:17:03,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:17:03,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:17:04,727 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:09,728 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:14,729 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:18,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:17:18,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:17:18,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:17:19,774 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:22,009 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:17:25,010 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:30,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:33,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:17:33,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:17:33,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:17:35,749 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:40,750 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:45,750 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:48,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:17:48,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:17:48,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:17:50,778 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:52,010 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:17:56,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:17:59,122 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:17:59,125 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:17:59,125 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:17:59,128 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:18:00,046 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:18:00,046 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:18:01,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:02,057 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:18:03,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:18:03,518 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:18:03,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:18:06,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:11,745 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:16,746 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:18,517 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:18:18,518 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:18:18,518 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:18:22,010 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:18:22,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:27,011 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:30,200 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:18:32,742 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:33,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:18:33,691 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:18:33,691 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:18:37,928 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:42,928 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:47,929 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:49,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:18:49,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:18:49,014 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:18:50,319 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:18:51,324 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:18:52,011 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:18:53,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:58,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:18:58,642 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:18:59,373 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:19:03,679 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:08,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:13,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:19:13,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:18,682 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:20,158 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:19:20,159 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:19:20,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:19:20,865 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:19:21,529 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:19:22,011 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:19:24,012 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:28,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:19:29,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:34,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:35,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:19:35,865 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:19:40,020 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:43,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:19:45,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:50,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:50,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:19:50,866 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:19:52,012 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:19:56,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:19:58,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:20:01,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:05,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:20:05,868 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:20:07,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:12,118 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:13,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:20:17,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:20,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:20:20,865 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:20:22,012 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:20:23,013 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:28,014 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:28,129 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:20:32,907 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:20:33,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:35,865 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:20:35,866 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:20:39,091 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:43,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:20:44,042 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:20:44,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:49,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:51,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:20:51,908 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:20:52,094 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:20:54,121 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:20:54,619 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:20:58,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:21:00,131 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:05,285 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:06,229 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:21:06,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:21:06,907 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:21:11,130 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:14,566 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:21:17,106 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:18,343 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:21:21,907 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:21:21,908 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:21:22,146 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:21:22,146 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:27,148 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:28,421 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:21:32,567 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:37,475 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:21:37,476 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:21:37,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:40,523 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:21:43,042 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:48,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:52,015 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:21:52,474 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:21:52,475 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:21:52,694 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:21:53,694 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:21:58,696 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:02,774 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:22:04,605 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:08,359 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:22:08,359 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:22:10,559 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:14,876 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:22:15,567 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:20,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:22,017 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:22:23,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:22:23,352 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:22:25,623 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:26,957 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:22:31,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:37,058 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:22:37,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:38,349 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:22:38,350 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:22:42,646 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:48,167 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:22:48,168 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:52,018 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:22:53,350 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:22:53,352 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:22:53,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:22:59,008 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:00,367 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:23:04,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:08,351 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:23:08,352 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:23:10,020 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:10,453 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:23:15,568 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:20,519 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:23:21,002 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:22,019 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:23:23,351 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:23:23,352 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:23:26,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:30,594 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:23:32,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:37,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:38,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:23:38,495 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:23:40,673 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:23:42,796 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:48,569 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:50,754 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:23:52,019 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:23:53,492 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:23:53,493 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:23:53,753 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:23:58,755 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:00,819 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:03,838 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:08,653 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:24:08,654 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:24:08,868 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:09,992 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:24:09,996 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:24:09,997 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:24:10,000 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:24:10,898 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:24:10,899 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:11,903 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:14,003 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:19,570 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:22,020 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:24:22,989 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:23,652 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:24:23,653 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:24:24,876 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:29,768 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:24:30,771 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:33,078 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:36,132 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:39,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:24:39,835 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:24:41,836 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:43,163 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:47,790 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:52,020 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:24:53,051 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:24:53,239 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:24:54,833 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:24:54,834 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:24:58,095 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:01,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:25:03,474 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:04,340 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:25:08,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:09,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:25:09,836 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:25:13,772 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:14,411 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:25:16,191 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:25:19,192 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:22,022 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:25:24,491 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:25:24,751 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:24,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:25:24,835 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:25:30,034 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:32,244 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:25:34,570 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:25:35,971 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:39,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:25:39,834 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:25:41,092 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:44,634 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:25:46,161 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:47,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:25:51,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:25:52,023 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:25:54,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:25:54,835 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:25:56,814 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:25:57,023 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:02,024 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:02,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:26:06,962 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:26:07,052 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:09,834 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:26:09,835 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:26:13,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:17,245 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:26:18,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:19,034 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:26:22,023 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:26:24,024 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:25,403 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:26:25,404 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:26:29,109 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:26:29,734 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:35,246 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:39,241 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:26:40,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:26:40,403 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:26:40,608 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:45,610 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:47,412 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:26:51,022 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:26:51,353 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:26:52,023 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:26:55,402 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:26:55,403 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:26:56,641 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:00,524 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:27:02,306 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:07,413 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:11,340 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:27:11,343 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:27:12,571 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:14,639 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:27:18,413 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:22,025 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:27:24,028 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:26,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:27:26,342 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:27:26,741 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:27:29,511 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:35,417 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:36,818 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:27:41,330 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:41,341 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:27:41,342 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:27:46,563 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:47,170 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:27:47,172 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:27:47,173 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:27:47,179 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:27:47,413 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:27:47,903 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:27:47,906 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:27:48,910 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:27:52,026 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:27:52,030 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:56,859 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:27:56,871 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:27:57,101 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:27:58,991 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:28:02,414 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:28:02,446 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:07,448 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:09,071 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:28:11,839 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:28:11,840 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:28:13,049 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:17,415 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: keepalive +2023-11-18 19:28:18,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:21,159 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:28:22,026 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:28:24,027 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:28,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:28:28,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:28:29,676 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:31,226 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:28:35,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:40,426 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:43,348 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:28:43,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:28:43,423 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:28:45,719 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:51,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:52,031 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:28:55,445 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:28:56,539 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:28:58,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:28:58,424 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:29:01,655 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:07,180 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:09,565 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:29:12,181 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:13,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:29:13,423 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:29:17,420 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:18,665 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:29:22,030 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:29:23,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:28,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:28,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:29:28,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:29:30,752 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:29:33,444 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:38,445 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:40,828 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:29:43,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:29:43,423 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:29:43,688 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:49,421 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:52,030 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:29:52,939 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:29:55,031 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:29:58,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:29:58,424 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:30:00,681 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:05,047 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:30:06,104 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:11,120 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:13,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:30:13,424 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:30:16,595 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:19,151 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:30:22,033 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:30:22,033 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:27,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:28,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:30:28,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:30:31,247 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:30:32,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:37,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:42,694 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:43,344 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:30:43,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:30:43,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:30:48,422 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:52,033 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:30:53,873 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:30:55,499 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:30:58,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:30:58,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:30:59,680 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:05,423 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:07,672 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:31:11,117 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:13,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:31:13,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:31:16,720 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:19,763 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:31:22,034 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:31:22,035 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:27,036 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:28,425 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:31:28,426 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:31:30,871 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:31:32,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:37,479 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:40,577 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: partial_history +2023-11-18 19:31:40,579 DEBUG SenderThread:60204 [sender.py:send():380] send: history +2023-11-18 19:31:40,579 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: summary_record +2023-11-18 19:31:40,581 INFO SenderThread:60204 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-18 19:31:40,940 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/wandb-summary.json +2023-11-18 19:31:40,941 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:31:42,591 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:42,953 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:31:43,424 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: stop_status +2023-11-18 19:31:43,425 DEBUG SenderThread:60204 [sender.py:send_request():407] send_request: stop_status +2023-11-18 19:31:48,477 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report +2023-11-18 19:31:52,035 DEBUG SenderThread:60204 [sender.py:send():380] send: stats +2023-11-18 19:31:53,039 INFO Thread-12 :60204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231118_132020-qg88991p/files/output.log +2023-11-18 19:31:54,036 DEBUG HandlerThread:60204 [handler.py:handle_request():146] handle_request: status_report