diff --git "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" --- "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" +++ "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" @@ -69438,3 +69438,6245 @@ 2023-11-20 22:28:35,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive 2023-11-20 22:28:37,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report 2023-11-20 22:28:40,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:45,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:46,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:28:48,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:53,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:53,450 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:28:55,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:58,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:58,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:29:00,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:03,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:06,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:07,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:29:08,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:11,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:13,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:16,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:16,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:17,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:29:19,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:21,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:23,451 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:29:24,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:27,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:29,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:29:29,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:32,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:34,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:37,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:39,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:42,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:43,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:29:45,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:47,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:50,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:53,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:29:53,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:29:55,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:29:55,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:29:58,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:00,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:03,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:03,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:04,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:30:04,024 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:30:04,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:30:04,027 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:30:04,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:30:05,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:30:06,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:06,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:30:10,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:11,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:15,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:16,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:18,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:30:20,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:21,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:23,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:30:26,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:26,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:30,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:30:31,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:31,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:36,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:36,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:40,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:30:41,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:42,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:46,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:47,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:52,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:30:52,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:52,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:30:53,453 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:30:57,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:30:58,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:02,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:02,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:03,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:04,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:31:08,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:08,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:13,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:13,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:14,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:31:18,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:18,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:23,454 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:31:23,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:31:24,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:24,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:29,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:29,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:34,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:34,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:35,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:31:39,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:40,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:44,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:45,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:47,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:31:50,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:50,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:53,455 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:31:55,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:31:56,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:31:58,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:32:00,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:01,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:05,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:07,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:07,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:10,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:32:11,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:12,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:16,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:19,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:19,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:19,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:21,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:22,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:32:23,457 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:32:24,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:26,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:30,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:31,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:34,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:32:35,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:36,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:40,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:41,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:45,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:46,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:32:47,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:50,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:52,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:53,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:32:55,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:32:57,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:32:58,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:00,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:02,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:06,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:06,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:07,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:08,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:11,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:12,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:17,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:17,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:19,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:22,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:22,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:23,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:33:27,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:28,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:32,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:32,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:34,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:37,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:39,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:43,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:44,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:45,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:48,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:50,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:53,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:33:53,460 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:33:54,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:33:54,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:33:54,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:33:54,681 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:33:55,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:33:55,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:33:56,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:57,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:33:58,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:00,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:03,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:05,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:08,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:34:09,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:11,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:14,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:16,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:20,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:20,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:34:22,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:23,460 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:34:25,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:27,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:30,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:34:30,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:32,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:35,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:37,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:41,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:41,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:42,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:42,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:34:46,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:47,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:51,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:51,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:53,461 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:34:53,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:34:54,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:34:56,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:34:58,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:03,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:03,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:06,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:35:08,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:09,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:13,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:14,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:16,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:35:18,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:19,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:23,463 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:35:23,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:23,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:24,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:27,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:35:28,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:29,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:34,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:34,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:36,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:35:39,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:40,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:45,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:45,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:48,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:35:50,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:51,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:53,464 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:35:55,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:55,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:35:56,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:35:58,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:36:00,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:02,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:06,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:07,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:08,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:36:11,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:12,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:16,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:17,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:20,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:36:21,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:23,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:23,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:36:26,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:28,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:30,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:36:31,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:33,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:36,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:38,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:41,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:42,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:36:44,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:46,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:49,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:52,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:52,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:36:53,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:36:54,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:36:57,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:36:59,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:03,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:04,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:04,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:37:08,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:09,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:13,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:14,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:16,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:37:18,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:20,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:23,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:37:24,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:24,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:25,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:29,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:37:29,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:31,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:35,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:36,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:37,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:37:37,308 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:37:37,309 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:37:37,311 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:37:38,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:37:39,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:37:40,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:41,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:46,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:50,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:37:52,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:37:52,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:53,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:37:57,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:37:57,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:02,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:38:02,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:03,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:07,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:08,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:12,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:13,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:16,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:38:17,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:18,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:22,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:23,469 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:38:24,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:26,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:38:28,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:29,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:33,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:35,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:38,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:38:40,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:43,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:46,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:48,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:50,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:38:51,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:38:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:53,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:38:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:38:58,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:00,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:39:03,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:03,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:08,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:08,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:14,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:14,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:14,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:39:19,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:19,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:23,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:39:24,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:24,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:24,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:25,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:39:29,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:29,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:34,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:35,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:37,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:39:39,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:40,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:45,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:45,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:47,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:39:50,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:50,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:53,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:39:55,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:55,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:39:56,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:39:59,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:40:00,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:01,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:05,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:06,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:10,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:40:11,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:12,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:17,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:17,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:20,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:40:22,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:23,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:23,472 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:40:28,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:28,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:30,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:40:33,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:33,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:38,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:38,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:38,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:38,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:40,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:40:43,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:43,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:48,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:49,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:52,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:40:53,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:40:54,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:54,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:40:59,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:40:59,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:04,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:04,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:09,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:09,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:14,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:14,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:19,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:19,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:23,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:41:24,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:24,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:24,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:25,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:29,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:41:29,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:31,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:34,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:36,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:39,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:41:40,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:41,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:45,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:46,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:50,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:41:50,256 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:41:50,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:41:50,263 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:41:50,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:41:50,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:51,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:41:52,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:41:53,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:41:55,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:41:57,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:01,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:02,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:03,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:42:06,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:07,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:12,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:13,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:13,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:42:17,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:18,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:22,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:23,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:23,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:42:24,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:42:27,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:28,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:33,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:33,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:36,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:42:38,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:39,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:43,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:48,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:42:49,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:49,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:53,477 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:42:54,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:55,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:42:58,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:42:59,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:42:59,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:01,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:04,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:06,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:10,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:10,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:11,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:43:12,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:15,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:17,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:20,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:21,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:43:22,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:23,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:43:25,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:28,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:31,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:31,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:43:34,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:36,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:39,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:41,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:41,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:41,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:44,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:45,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:43:46,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:49,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:52,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:43:53,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:43:55,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:43:55,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:43:57,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:00,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:02,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:05,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:44:05,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:10,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:12,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:12,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:12,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:15,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:44:16,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:17,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:21,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:22,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:23,480 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:44:25,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:44:26,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:27,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:31,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:32,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:36,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:37,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:44:38,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:41,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:43,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:46,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:46,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:44:48,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:51,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:53,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:53,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:44:57,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:44:58,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:44:58,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:02,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:03,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:07,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:09,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:09,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:12,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:14,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:17,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:19,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:19,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:22,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:23,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:45:24,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:28,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:29,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:33,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:34,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:36,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:45:36,182 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:45:36,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:45:36,185 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:45:36,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:45:37,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:38,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:39,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:39,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:44,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:44,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:49,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:49,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:50,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:53,482 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:45:54,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:45:55,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:45:59,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:45:59,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:00,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:04,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:05,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:10,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:11,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:11,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:46:15,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:16,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:20,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:21,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:21,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:46:23,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:46:26,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:26,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:31,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:31,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:46:32,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:32,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:36,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:37,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:42,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:43,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:43,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:46:47,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:48,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:52,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:53,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:46:53,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:46:53,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:46:58,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:46:58,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:03,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:03,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:47:04,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:08,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:09,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:14,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:14,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:15,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:47:19,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:19,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:23,484 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:47:24,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:25,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:47:25,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:29,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:30,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:35,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:35,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:47:35,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:40,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:40,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:45,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:45,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:47:46,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:50,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:51,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:47:53,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:47:55,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:47:55,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:47:56,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:00,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:01,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:05,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:06,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:07,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:48:10,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:11,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:15,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:48:16,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:16,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:21,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:21,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:48:22,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:23,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:48:26,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:28,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:29,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:48:31,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:33,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:37,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:38,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:39,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:48:42,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:43,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:47,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:47,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:48,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:51,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:48:53,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:48:53,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:48:53,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:48:58,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:01,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:03,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:04,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:49:08,950 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:49:08,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:49:08,953 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:49:09,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:09,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:49:09,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:09,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:11,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:14,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:14,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:20,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:20,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:21,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:23,490 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:49:25,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:26,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:31,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:32,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:32,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:36,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:37,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:41,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:42,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:43,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:46,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:48,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:51,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:49:52,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:49:53,489 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:49:53,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:49:58,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:01,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:50:03,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:04,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:08,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:09,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:11,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:50:13,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:14,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:19,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:20,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:23,489 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:50:23,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:50:24,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:25,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:29,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:33,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:50:34,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:36,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:39,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:42,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:45,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:45,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:50:47,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:50,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:50:53,491 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:50:55,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:50:55,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:50:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:00,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:03,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:03,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:06,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:07,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:51:08,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:11,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:13,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:16,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:18,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:20,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:51:22,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:23,491 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:51:23,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:27,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:28,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:32,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:51:33,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:38,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:38,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:42,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:51:43,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:43,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:48,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:50,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:53,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:51:53,492 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:51:53,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:51:55,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:51:58,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:01,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:01,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:05,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:52:06,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:09,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:12,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:14,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:15,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:52:17,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:19,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:22,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:23,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:52:24,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:28,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:29,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:52:30,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:33,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:35,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:38,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:39,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:52:40,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:43,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:45,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:49,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:49,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:52:49,541 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:52:49,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:52:49,546 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:52:49,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:52:50,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:50,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:52:51,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:52:53,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:52:54,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:52:56,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:52:59,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:02,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:53:02,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:04,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:07,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:09,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:12,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:14,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:53:14,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:17,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:19,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:22,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:23,495 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:53:25,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:26,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:53:27,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:30,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:32,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:36,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:37,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:38,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:53:41,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:42,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:46,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:47,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:49,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:53:51,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:52,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:53,496 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:53:57,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:53:58,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:53:59,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:54:02,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:03,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:08,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:09,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:09,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:54:13,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:14,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:19,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:19,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:19,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:21,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:54:23,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:54:24,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:24,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:29,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:29,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:31,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:54:34,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:35,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:40,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:41,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:44,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:54:45,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:46,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:50,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:52,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:54:53,498 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:54:56,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:54:56,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:54:57,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:01,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:03,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:03,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:06,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:55:06,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:08,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:11,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:14,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:16,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:18,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:55:19,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:21,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:23,500 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:55:25,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:26,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:28,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:55:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:31,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:35,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:36,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:40,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:40,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:55:42,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:45,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:47,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:50,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:50,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:55:52,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:55:53,501 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:55:55,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:55:58,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:01,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:01,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:03,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:06,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:08,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:11,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:11,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:14,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:16,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:19,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:21,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:22,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:23,501 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:56:24,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:27,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:29,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:32,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:56:32,377 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:56:32,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:56:32,381 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:56:32,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:33,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:56:34,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:35,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:35,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:40,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:43,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:45,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:46,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:48,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:50,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:53,502 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:56:54,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:56:55,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:56:58,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:56:59,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:01,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:05,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:06,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:10,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:57:10,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:11,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:16,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:17,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:17,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:17,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:20,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:57:21,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:22,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:23,503 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:57:27,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:28,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:32,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:57:32,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:33,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:37,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:39,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:42,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:57:43,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:44,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:48,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:49,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:53,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:57:54,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:54,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:57:54,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:57:59,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:57:59,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:03,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:58:05,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:05,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:05,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:10,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:10,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:15,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:15,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:15,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:58:20,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:20,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:23,505 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:58:25,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:25,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:25,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:58:30,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:30,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:35,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:36,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:58:36,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:36,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:40,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:41,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:45,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:47,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:50,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:58:51,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:53,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:58:53,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:53,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:58:57,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:58:58,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:02,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:59:03,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:08,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:09,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:12,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:59:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:18,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:19,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:23,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:59:24,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:59:24,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:25,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:29,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:32,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:34,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:36,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:59:37,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:39,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:43,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:44,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:46,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:59:48,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:49,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:53,508 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:59:54,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:59:55,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:59:58,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:59:59,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:00,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:05,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:09,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:10,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:00:10,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:14,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:15,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:19,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:00:19,296 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:00:19,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:00:19,299 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:00:19,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:20,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:00:20,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:00:21,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:00:21,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:23,508 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:00:24,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:26,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:30,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:31,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:32,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:00:35,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:36,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:40,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:41,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:44,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:00:45,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:46,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:50,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:51,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:00:53,511 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:00:55,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:00:56,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:00:56,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:00,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:01,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:05,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:05,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:06,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:08,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:01:10,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:11,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:15,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:17,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:18,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:01:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:22,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:23,511 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:01:26,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:27,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:30,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:01:31,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:32,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:37,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:37,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:37,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:40,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:01:42,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:43,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:47,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:48,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:53,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:53,512 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:01:53,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:01:54,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:01:58,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:01:58,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:03,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:03,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:08,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:10,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:02:13,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:14,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:18,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:19,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:22,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:02:23,512 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:02:24,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:24,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:29,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:29,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:32,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:02:34,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:35,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:40,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:40,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:44,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:02:45,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:45,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:50,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:51,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:02:53,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:02:54,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:02:55,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:02:56,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:00,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:01,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:05,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:06,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:08,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:03:10,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:11,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:15,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:16,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:18,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:03:21,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:21,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:23,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:03:26,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:27,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:30,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:03:32,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:32,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:37,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:38,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:38,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:43,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:03:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:47,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:48,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:52,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:53,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:03:54,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:03:55,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:03:57,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:03:59,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:02,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:05,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:07,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:08,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:04:10,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:12,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:16,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:17,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:18,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:04:18,334 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:04:18,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:04:18,338 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:04:18,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:04:19,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:04:20,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:04:21,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:23,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:23,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:04:28,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:28,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:28,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:32,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:04:33,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:34,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:39,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:40,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:40,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:40,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:42,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:04:44,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:45,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:51,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:04:53,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:04:53,666 WARNING FileStreamThread:607337 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream. func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 4326, 'content': ['{"system.gpu.0.memoryAllocated": 79.71, "system.gpu.process.0.memoryAllocated": 79.71, "system.gpu.0.memoryAllocatedBytes": 13687805815.47, "system.gpu.process.0.memoryAllocatedBytes": 13687805815.47, "system.gpu.0.memory": 20.0, "system.gpu.process.0.memory": 20.0, "system.gpu.0.gpu": 48.67, "system.gpu.process.0.gpu": 48.67, "system.gpu.0.temp": 44.47, "system.gpu.process.0.temp": 44.47, "system.gpu.0.powerWatts": 121.37, "system.gpu.process.0.powerWatts": 121.37, "system.gpu.0.powerPercent": 35.73, "system.gpu.process.0.powerPercent": 35.73, "system.network.sent": 40079727480.87, "system.network.recv": 108455167699.8, "system.proc.memory.availableMB": 3500.83, "system.memory": 93.76, "system.proc.memory.rssMB": 48327.5, "system.proc.memory.percent": 85.92, "system.cpu": 27.34, "system.cpu.0.cpu_percent": 25.13, "system.cpu.1.cpu_percent": 51.91, "system.cpu.2.cpu_percent": 33.92, "system.cpu.3.cpu_percent": 31.69, "system.cpu.4.cpu_percent": 28.47, "system.cpu.5.cpu_percent": 25.37, "system.cpu.6.cpu_percent": 33.07, "system.cpu.7.cpu_percent": 25.93, "system.proc.cpu.threads": 60, "system.disk.\\\\.usagePercent": 3.9, "system.disk.\\\\.usageGB": 37.45, "system.disk.in": 2047.67, "system.disk.out": 8520.88, "_wandb": true, "_timestamp": 1700514203.513119, "_runtime": 129854.479032}', '{"system.gpu.0.memoryAllocated": 79.54, "system.gpu.process.0.memoryAllocated": 79.54, "system.gpu.0.memoryAllocatedBytes": 13658972160.0, "system.gpu.process.0.memoryAllocatedBytes": 13658972160.0, "system.gpu.0.memory": 23.33, "system.gpu.process.0.memory": 23.33, "system.gpu.0.gpu": 53.2, "system.gpu.process.0.gpu": 53.2, "system.gpu.0.temp": 44.8, "system.gpu.process.0.temp": 44.8, "system.gpu.0.powerWatts": 104.02, "system.gpu.process.0.powerWatts": 104.02, "system.gpu.0.powerPercent": 30.57, "system.gpu.process.0.powerPercent": 30.57, "system.network.sent": 40080350904.2, "system.network.recv": 108459703433.87, "system.proc.memory.availableMB": 3462.37, "system.memory": 93.83, "system.proc.memory.rssMB": 48362.06, "system.proc.memory.percent": 85.99, "system.cpu": 29.14, "system.cpu.0.cpu_percent": 26.6, "system.cpu.1.cpu_percent": 53.33, "system.cpu.2.cpu_percent": 32.67, "system.cpu.3.cpu_percent": 26.36, "system.cpu.4.cpu_percent": 31.14, "system.cpu.5.cpu_percent": 35.77, "system.cpu.6.cpu_percent": 47.15, "system.cpu.7.cpu_percent": 19.83, "system.proc.cpu.threads": 60, "system.disk.\\\\.usagePercent": 3.9, "system.disk.\\\\.usageGB": 37.45, "system.disk.in": 2047.67, "system.disk.out": 8521.32, "_wandb": true, "_timestamp": 1700514233.513694, "_runtime": 129884.479607}']}, 'output.log': {'offset': 827, 'content': ['ERROR 2023-11-20T21:03:42.213619 30%|███████████████████████████▋ | 18237/60000 [36:04:25<134:53:03, 11.63s/it]\n']}}, 'dropped': 0}} +2023-11-20 23:04:54,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:04:55,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:04:56,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:00,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:01,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:04,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:05:05,620 WARNING FileStreamThread:607337 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream. func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream',), kwargs: {'json': {'files': {'output.log': {'offset': 827, 'content': ['ERROR 2023-11-20T21:04:54.301230 30%|███████████████████████████▋ | 18243/60000 [36:05:38<134:33:27, 11.60s/it]\n']}}, 'dropped': 0}} +2023-11-20 23:05:05,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:06,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:10,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:11,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:16,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:16,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:16,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:05:21,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:21,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:23,519 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:05:26,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:27,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:29,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:05:31,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:32,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:36,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:37,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:41,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:05:42,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:42,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:47,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:48,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:51,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:05:52,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:53,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:05:53,517 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:05:58,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:05:59,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:02,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:06:03,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:04,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:08,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:09,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:13,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:14,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:06:15,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:15,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:18,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:20,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:23,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:06:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:26,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:26,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:06:29,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:31,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:34,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:36,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:06:36,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:40,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:41,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:45,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:46,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:06:47,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:51,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:52,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:53,519 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:06:56,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:06:58,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:06:58,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:07:01,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:03,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:06,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:08,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:11,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:07:12,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:13,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:17,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:19,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:22,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:23,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:07:23,520 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:07:24,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:27,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:29,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:29,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:33,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:33,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:07:34,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:38,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:40,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:43,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:07:43,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:45,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:51,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:53,523 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:07:54,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:07:54,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:07:56,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:07:59,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:01,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:03,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:08:03,282 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:08:03,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:08:03,287 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:08:03,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:08:04,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:08:05,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:05,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:08:06,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:10,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:11,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:15,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:16,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:08:17,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:20,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:22,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:23,524 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:08:26,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:28,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:28,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:08:31,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:33,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:36,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:38,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:08:39,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:42,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:44,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:47,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:49,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:51,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:08:52,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:08:53,524 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:08:54,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:08:58,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:00,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:01,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:09:03,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:05,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:08,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:11,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:13,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:15,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:09:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:18,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:22,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:23,526 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:09:23,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:25,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:09:27,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:28,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:33,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:33,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:37,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:09:38,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:39,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:44,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:44,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:44,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:44,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:47,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:09:49,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:50,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:53,528 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:09:55,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:09:55,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:09:58,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:10:00,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:01,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:05,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:06,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:11,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:11,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:12,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:10:16,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:16,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:22,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:22,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:22,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:10:23,527 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:10:27,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:27,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:32,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:32,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:32,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:10:33,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:37,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:38,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:42,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:43,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:45,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:10:47,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:48,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:53,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:10:53,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:53,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:10:57,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:10:58,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:10:58,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:03,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:03,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:03,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:07,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:11:08,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:09,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:14,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:14,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:17,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:11:19,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:20,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:23,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:11:25,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:25,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:27,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:11:30,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:31,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:35,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:36,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:39,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:11:40,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:41,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:45,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:46,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:49,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:11:49,136 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:11:49,137 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:11:49,140 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:11:49,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:11:50,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:51,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:11:52,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:11:53,531 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:11:55,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:11:57,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:01,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:02,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:03,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:12:06,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:07,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:12,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:12,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:15,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:12:17,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:18,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:22,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:23,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:23,533 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:12:27,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:12:27,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:28,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:32,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:33,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:38,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:38,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:39,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:12:43,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:43,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:48,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:48,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:48,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:49,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:12:53,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:53,533 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:12:54,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:12:58,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:12:59,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:01,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:13:03,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:04,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:08,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:09,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:13,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:14,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:18,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:19,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:23,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:13:23,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:25,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:27,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:13:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:30,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:35,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:39,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:39,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:13:40,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:44,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:45,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:49,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:49,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:50,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:51,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:13:53,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:13:54,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:13:56,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:13:59,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:01,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:03,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:14:04,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:06,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:10,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:11,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:13,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:14:15,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:16,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:21,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:21,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:21,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:23,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:14:25,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:14:26,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:27,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:32,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:32,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:35,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:14:38,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:38,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:42,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:43,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:48,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:48,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:49,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:49,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:14:53,536 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:14:53,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:54,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:14:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:14:59,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:15:00,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:05,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:08,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:10,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:11,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:15:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:15,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:19,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:19,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:19,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:20,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:21,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:15:23,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:15:24,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:25,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:30,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:30,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:33,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:15:35,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:35,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:40,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:40,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:45,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:45,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:15:46,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:50,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:51,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:53,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:15:55,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:15:55,432 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:15:55,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:15:55,436 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:15:55,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:15:56,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:15:56,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:15:57,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:16:01,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:01,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:06,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:06,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:08,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:16:11,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:12,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:17,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:17,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:18,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:16:22,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:22,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:23,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:16:27,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:27,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:29,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:16:32,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:33,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:37,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:38,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:41,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:16:42,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:44,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:48,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:49,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:51,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:16:53,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:53,540 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:16:54,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:16:58,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:16:59,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:01,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:17:03,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:05,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:09,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:10,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:11,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:17:14,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:15,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:19,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:19,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:19,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:20,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:23,541 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:17:23,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:17:24,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:26,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:29,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:31,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:35,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:35,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:17:37,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:40,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:42,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:45,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:45,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:17:47,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:50,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:52,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:17:53,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:17:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:17:57,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:17:58,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:03,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:05,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:07,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:18:08,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:11,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:13,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:17,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:18,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:18:19,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:22,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:23,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:18:25,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:27,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:30,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:18:30,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:32,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:35,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:37,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:40,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:18:40,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:42,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:45,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:47,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:51,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:52,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:18:53,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:18:53,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:18:56,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:18:58,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:01,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:19:01,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:03,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:06,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:08,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:11,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:14,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:16,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:17,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:19:19,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:21,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:23,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:19:25,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:26,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:27,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:19:30,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:31,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:35,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:36,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:39,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:19:39,334 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:19:39,334 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:19:39,337 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:19:39,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:19:40,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:19:41,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:41,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:19:42,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:46,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:47,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:51,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:51,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:19:52,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:19:53,546 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:19:56,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:19:58,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:01,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:20:02,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:04,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:07,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:09,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:12,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:20:12,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:14,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:17,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:19,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:22,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:20:22,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:23,547 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:20:25,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:27,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:30,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:32,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:34,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:20:36,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:37,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:41,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:46,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:20:46,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:51,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:53,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:20:54,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:54,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:20:57,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:20:58,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:20:59,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:02,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:08,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:21:08,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:10,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:13,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:15,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:18,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:21:19,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:20,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:23,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:21:24,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:25,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:29,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:21:30,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:31,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:35,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:36,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:40,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:41,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:43,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:21:45,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:46,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:50,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:51,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:21:53,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:21:53,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:21:56,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:21:56,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:01,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:02,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:05,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:22:07,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:07,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:12,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:13,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:16,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:22:17,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:18,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:22,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:23,551 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:22:24,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:24,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:28,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:22:28,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:29,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:33,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:35,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:38,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:22:38,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:40,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:43,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:45,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:48,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:50,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:22:50,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:53,552 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:22:54,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:22:55,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:22:59,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:01,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:02,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:23:05,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:06,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:10,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:11,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:14,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:23:15,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:16,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:20,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:21,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:21,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:21,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:23,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:23:23,245 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:23:23,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:23:23,248 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:23:23,563 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:23:23,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:23:24,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:23:26,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:26,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:31,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:32,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:35,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:23:36,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:37,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:41,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:43,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:45,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:23:47,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:48,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:52,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:53,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:23:54,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:54,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:23:57,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:23:57,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:23:59,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:02,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:05,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:07,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:24:08,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:10,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:13,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:15,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:18,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:20,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:24:20,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:23,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:24:24,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:25,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:30,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:24:30,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:31,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:35,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:36,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:40,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:41,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:42,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:24:45,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:46,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:52,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:24:53,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:24:54,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:24:56,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:24:57,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:01,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:02,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:06,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:07,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:11,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:12,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:16,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:17,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:21,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:23,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:23,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:23,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:23,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:25:26,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:25:27,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:28,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:32,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:33,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:37,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:37,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:25:39,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:42,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:44,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:47,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:49,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:25:49,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:53,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:25:53,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:25:54,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:25:58,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:00,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:26:00,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:04,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:05,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:09,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:10,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:12,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:26:15,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:15,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:20,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:21,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:23,558 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:26:24,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:26:25,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:26,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:30,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:31,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:34,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:26:36,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:36,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:41,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:41,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:41,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:44,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:26:46,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:46,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:51,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:26:52,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:53,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:26:54,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:26:57,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:26:57,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:02,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:02,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:06,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:27:07,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:08,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:13,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:13,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:13,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:16,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:27:18,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:18,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:23,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:23,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:27:24,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:29,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:27:29,439 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:27:29,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:27:29,442 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:27:29,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:27:30,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:30,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:27:34,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:35,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:39,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:39,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:41,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:42,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:27:44,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:46,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:50,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:52,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:27:52,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:27:53,560 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:27:55,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:27:57,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:01,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:02,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:28:02,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:06,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:07,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:12,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:12,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:12,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:14,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:28:17,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:17,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:22,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:23,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:23,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:28:26,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:28:28,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:28,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:33,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:35,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:38,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:28:38,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:40,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:43,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:46,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:48,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:28:48,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:51,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:53,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:28:53,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:28:57,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:28:58,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:00,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:02,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:03,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:07,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:08,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:09,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:12,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:14,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:14,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:17,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:19,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:23,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:23,563 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:29:25,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:26,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:28,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:30,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:33,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:36,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:36,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:38,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:41,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:44,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:46,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:46,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:49,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:51,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:53,564 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:29:54,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:29:56,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:29:58,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:29:59,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:01,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:04,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:06,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:09,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:11,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:30:12,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:14,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:17,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:20,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:21,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:30:22,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:23,565 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:30:25,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:28,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:30,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:31,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:30:34,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:35,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:39,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:40,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:30:41,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:44,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:46,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:49,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:51,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:30:52,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:30:53,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:30:55,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:30:56,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:00,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:02,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:02,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:31:02,489 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:31:02,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:31:02,494 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:31:03,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:31:04,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:05,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:06,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:07,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:11,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:12,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:12,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:14,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:16,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:17,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:21,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:22,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:23,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:31:24,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:27,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:27,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:32,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:32,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:34,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:37,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:38,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:43,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:43,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:43,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:46,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:48,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:49,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:53,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:31:54,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:54,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:54,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:31:54,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:31:56,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:31:59,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:00,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:04,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:05,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:06,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:32:09,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:10,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:14,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:16,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:32:19,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:21,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:23,568 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:32:24,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:24,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:27,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:29,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:32:29,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:32,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:34,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:37,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:40,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:41,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:32:42,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:45,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:47,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:51,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:32:53,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:32:53,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:32:56,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:32:58,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:01,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:03,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:33:03,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:06,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:08,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:11,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:11,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:13,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:33:14,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:16,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:19,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:21,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:23,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:33:24,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:25,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:33:26,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:29,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:32,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:34,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:33:34,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:37,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:39,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:42,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:42,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:44,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:33:45,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:47,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:50,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:52,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:33:53,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:33:55,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:33:56,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:33:57,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:01,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:02,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:06,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:06,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:34:07,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:11,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:12,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:16,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:18,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:18,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:34:21,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:23,572 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:34:23,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:27,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:29,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:34:29,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:32,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:34,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:37,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:38,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:34:38,209 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:34:38,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:34:38,212 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:34:39,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:34:39,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:34:40,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:43,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:45,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:48,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:50,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:51,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:34:53,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:34:53,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:34:55,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:34:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:01,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:35:04,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:06,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:09,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:11,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:13,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:35:15,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:16,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:20,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:22,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:23,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:35:23,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:35:25,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:27,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:32,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:35,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:38,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:41,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:41,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:35:43,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:46,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:48,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:51,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:53,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:35:54,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:35:54,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:35:56,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:35:59,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:01,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:04,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:06,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:36:07,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:09,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:12,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:14,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:14,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:16,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:36:17,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:19,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:22,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:23,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:36:25,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:26,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:36:27,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:30,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:32,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:36,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:37,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:39,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:36:41,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:42,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:46,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:47,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:51,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:52,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:36:53,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:36:53,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:36:56,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:36:58,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:02,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:03,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:37:04,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:07,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:09,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:12,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:13,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:37:15,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:17,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:20,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:23,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:23,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:37:23,577 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:37:25,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:28,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:30,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:33,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:35,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:37:36,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:38,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:41,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:43,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:43,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:45,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:37:46,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:48,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:51,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:53,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:37:55,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:37:56,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:37:59,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:00,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:01,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:06,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:06,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:09,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:11,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:12,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:16,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:17,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:19,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:21,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:22,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:23,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:38:26,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:27,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:27,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:38:27,964 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:38:27,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:38:27,967 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:38:28,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:38:28,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:29,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:31,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:32,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:37,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:37,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:39,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:42,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:43,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:47,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:48,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:51,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:38:53,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:53,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:38:53,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:38:58,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:38:58,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:03,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:03,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:05,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:39:08,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:08,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:14,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:15,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:39:19,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:19,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:23,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:39:24,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:24,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:25,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:39:29,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:29,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:34,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:34,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:37,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:39:39,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:40,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:45,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:45,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:45,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:49,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:39:50,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:50,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:39:53,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:39:55,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:39:55,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:00,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:01,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:01,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:01,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:40:05,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:06,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:10,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:12,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:13,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:40:16,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:17,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:21,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:22,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:23,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:40:23,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:40:26,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:27,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:31,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:33,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:33,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:35,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:40:37,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:38,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:42,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:43,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:47,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:40:48,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:48,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:53,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:40:53,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:53,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:40:57,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:40:58,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:40:58,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:03,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:04,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:08,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:09,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:11,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:41:14,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:14,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:19,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:19,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:23,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:41:23,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:41:24,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:24,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:29,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:31,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:31,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:33,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:41:34,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:36,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:39,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:42,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:45,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:45,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:41:47,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:50,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:53,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:41:54,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:41:55,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:41:57,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:41:59,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:00,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:04,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:09,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:10,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:14,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:42:14,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:15,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:19,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:20,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:23,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:42:23,301 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:42:23,301 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:42:23,304 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:42:23,586 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:42:24,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:42:24,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:26,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:42:26,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:29,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:31,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:35,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:36,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:42:36,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:40,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:41,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:45,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:45,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:46,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:47,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:42:50,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:51,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:42:53,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:42:56,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:42:57,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:42:57,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:01,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:02,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:07,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:07,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:11,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:43:12,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:13,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:18,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:18,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:21,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:43:23,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:23,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:43:23,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:28,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:28,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:31,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:43:33,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:34,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:39,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:39,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:41,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:43:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:44,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:49,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:49,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:43:51,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:43:53,593 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:43:54,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:43:55,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:00,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:01,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:01,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:01,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:03,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:44:06,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:06,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:11,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:11,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:16,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:44:16,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:16,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:21,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:21,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:23,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:44:26,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:27,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:28,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:44:31,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:32,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:37,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:37,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:38,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:44:42,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:42,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:47,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:48,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:44:48,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:52,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:53,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:53,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:44:58,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:44:58,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:44:59,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:45:03,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:03,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:08,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:08,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:11,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:45:13,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:13,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:18,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:19,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:21,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:45:23,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:45:24,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:24,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:29,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:29,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:31,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:45:34,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:35,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:40,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:40,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:43,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:45:45,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:45,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:50,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:51,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:45:53,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:45:53,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:45:55,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:45:56,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:00,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:02,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:02,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:02,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:03,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:46:03,805 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:46:03,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:46:03,808 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:46:03,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:46:05,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:05,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:46:07,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:10,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:12,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:16,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:46:16,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:17,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:21,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:23,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:23,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:46:26,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:46:26,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:31,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:33,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:33,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:33,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:36,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:46:37,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:38,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:42,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:43,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:46,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:46:47,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:48,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:52,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:53,599 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:46:54,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:46:58,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:46:58,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:46:59,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:03,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:05,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:05,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:08,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:47:09,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:10,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:14,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:16,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:18,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:47:19,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:21,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:23,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:47:25,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:27,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:29,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:47:30,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:32,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:35,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:37,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:39,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:47:41,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:46,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:48,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:49,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:47:51,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:53,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:47:53,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:47:56,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:47:59,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:01,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:48:01,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:04,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:10,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:11,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:48:12,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:15,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:17,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:21,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:21,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:22,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:48:23,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:23,601 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:48:26,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:28,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:31,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:33,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:34,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:48:36,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:38,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:41,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:43,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:44,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:48:46,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:48,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:53,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:48:54,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:48:54,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:48:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:48:59,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:03,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:05,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:06,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:49:08,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:10,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:13,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:15,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:16,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:49:18,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:20,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:23,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:23,604 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:49:25,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:26,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:49:28,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:30,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:34,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:34,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:49:34,417 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:49:34,418 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:49:34,421 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:49:34,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:49:36,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:36,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:49:39,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:41,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:44,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:45,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:49:46,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:49,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:51,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:49:53,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:49:55,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:49:55,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:49:57,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:02,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:05,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:05,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:07,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:50:08,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:11,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:13,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:16,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:18,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:50:18,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:21,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:23,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:50:24,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:26,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:28,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:50:29,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:31,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:34,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:36,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:38,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:50:40,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:41,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:45,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:46,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:48,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:50:51,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:51,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:50:53,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:50:56,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:50:57,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:00,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:51:02,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:02,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:07,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:08,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:08,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:12,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:51:12,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:13,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:17,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:19,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:22,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:51:22,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:23,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:51:24,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:28,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:30,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:32,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:51:34,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:35,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:39,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:40,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:40,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:42,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:51:45,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:45,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:50,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:50,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:53,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:51:55,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:51:55,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:51:58,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:52:00,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:00,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:05,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:05,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:05,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:08,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:52:10,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:11,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:15,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:16,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:18,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:52:21,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:21,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:23,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:52:26,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:26,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:28,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:52:31,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:31,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:36,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:37,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:38,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:52:42,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:42,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:47,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:47,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:50,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:52:52,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:52,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:52:53,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:52:58,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:52:58,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:00,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:53:03,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:03,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:08,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:08,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:08,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:08,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:53:08,644 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:53:08,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:53:08,648 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:53:09,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:53:10,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:53:13,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:13,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:19,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:19,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:20,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:53:23,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:53:24,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:29,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:30,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:30,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:53:34,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:35,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:40,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:40,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:53:40,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:45,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:46,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:50,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:53:51,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:53:51,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:53,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:53:57,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:53:58,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:00,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:54:03,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:03,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:08,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:08,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:08,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:10,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:54:13,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:13,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:19,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:19,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:20,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:54:23,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:54:24,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:30,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:30,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:30,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:54:35,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:35,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:40,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:40,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:54:41,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:45,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:46,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:50,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:51,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:54:53,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:54:55,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:54:55,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:54:56,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:01,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:02,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:03,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:55:06,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:07,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:12,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:12,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:13,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:55:13,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:17,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:18,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:22,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:22,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:55:23,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:55:23,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:27,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:28,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:32,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:55:32,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:34,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:37,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:39,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:42,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:55:43,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:44,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:48,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:49,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:53,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:55:53,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:55:54,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:55:55,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:55:58,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:00,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:03,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:04,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:56:06,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:08,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:11,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:14,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:14,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:56:16,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:19,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:21,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:23,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:56:24,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:24,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:56:27,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:29,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:32,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:33,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 23:56:33,215 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 23:56:33,216 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 23:56:33,220 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 23:56:33,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 23:56:34,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:56:34,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:38,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:39,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:43,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:45,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:46,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:56:48,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:50,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:53,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:56:53,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:56:55,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:56:59,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:56:59,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:00,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:04,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:06,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:07,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:57:09,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:11,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:14,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:16,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:19,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:57:19,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:21,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:23,615 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:57:25,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:27,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:29,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:57:31,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:32,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:37,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:37,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:39,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:57:42,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:43,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:48,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:48,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:57:49,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:53,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:57:53,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:57:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:57:58,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:57:59,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:00,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:04,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:06,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:08,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:58:09,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:11,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:14,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:16,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:18,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:58:19,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:21,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:23,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:58:25,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:27,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:27,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:28,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:58:30,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:32,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:35,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:37,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:38,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:58:41,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:42,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:46,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:58:46,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:48,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:51,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:53,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:58:53,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:58:57,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:58:58,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:02,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:59:03,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:03,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:08,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:08,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:13,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:14,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:15,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:59:18,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:19,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:23,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:59:24,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:24,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:25,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:59:29,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:30,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:35,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:59:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:35,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:40,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:41,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:41,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:43,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:59:45,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:46,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:50,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:52,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 23:59:53,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 23:59:53,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 23:59:56,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 23:59:57,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:01,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:03,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:00:03,221 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:00:03,222 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:00:03,225 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:00:03,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:00:03,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:05,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:00:07,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:08,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:12,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:14,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:15,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:00:18,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:19,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:23,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:23,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:00:24,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:25,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:00:28,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:31,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:34,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:35,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:00:36,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:39,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:41,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:41,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:43,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:00:45,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:46,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:50,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:52,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:00:53,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:00:54,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:00:55,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:00:57,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:00,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:04,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:01:06,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:07,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:11,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:13,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:14,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:01:17,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:18,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:22,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:23,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:23,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:01:24,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:01:27,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:28,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:32,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:01:33,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:34,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:38,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:39,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:43,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:44,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:44,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:01:48,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:49,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:53,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:01:53,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:01:54,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:01:55,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:01:58,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:01,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:03,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:02:04,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:06,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:09,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:11,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:11,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:13,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:02:14,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:16,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:19,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:23,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:02:23,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:02:24,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:26,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:29,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:32,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:33,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:02:35,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:37,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:40,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:42,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:43,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:02:45,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:47,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:50,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:52,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:52,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:02:53,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:02:53,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:02:55,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:02:59,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:01,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:01,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:03:04,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:06,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:09,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:09,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:11,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:03:11,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:14,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:16,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:19,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:03:19,257 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:03:19,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:03:19,259 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:03:19,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:03:20,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:21,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:03:22,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:23,622 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:03:25,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:27,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:30,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:32,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:33,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:03:35,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:37,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:41,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:41,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:03:43,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:46,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:48,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:51,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:03:52,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:53,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:03:53,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:03:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:03:58,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:00,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:04:03,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:04,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:10,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:10,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:10,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:13,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:04:14,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:15,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:19,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:20,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:23,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:04:23,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:04:25,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:25,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:30,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:31,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:35,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:04:35,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:36,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:40,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:42,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:42,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:45,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:04:46,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:47,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:51,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:52,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:04:53,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:04:55,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:04:56,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:04:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:01,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:02,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:05,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:05:07,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:12,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:13,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:17,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:05:17,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:18,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:22,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:23,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:05:24,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:25,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:05:28,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:29,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:33,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:35,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:38,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:39,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:05:40,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:43,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:45,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:48,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:49,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:05:50,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:53,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:05:54,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:05:56,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:56,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:05:57,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:00,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:01,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:05,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:06,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:08,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:10,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:11,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:15,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:17,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:17,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:21,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:22,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:23,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:06:26,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:27,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:27,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:31,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:33,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:36,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:37,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:38,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:41,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:44,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:44,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:44,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:06:44,154 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:06:44,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:06:44,157 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:06:44,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:06:45,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:47,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:49,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:52,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:53,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:06:54,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:06:55,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:06:57,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:06:59,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:02,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:04,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:08,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:09,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:07:09,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:14,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:18,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:19,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:07:20,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:23,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:07:24,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:29,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:07:29,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:30,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:34,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:36,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:37,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:07:40,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:41,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:45,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:46,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:47,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:07:50,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:51,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:53,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:07:56,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:07:56,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:07:57,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:01,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:02,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:07,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:07,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:08,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:12,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:14,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:14,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:17,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:17,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:22,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:23,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:08:24,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:25,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:27,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:29,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:32,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:34,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:35,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:37,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:39,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:42,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:46,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:47,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:49,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:52,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:08:53,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:08:55,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:08:56,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:08:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:01,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:01,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:04,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:09:04,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:06,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:09,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:11,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:14,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:09:14,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:16,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:19,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:22,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:23,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:09:24,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:09:25,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:27,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:30,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:32,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:33,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:09:36,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:37,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:41,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:42,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:43,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:09:46,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:47,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:51,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:53,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:09:53,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:09:55,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:09:57,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:09:58,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:02,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:03,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:10:03,896 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:10:03,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:10:03,900 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:10:04,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:04,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:10:05,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:10:07,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:09,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:12,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:14,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:14,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:15,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:10:18,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:19,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:23,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:23,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:10:25,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:25,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:10:28,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:30,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:33,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:35,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:10:36,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:41,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:46,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:46,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:48,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:10:49,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:51,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:53,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:10:54,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:10:56,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:10:58,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:10:59,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:01,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:04,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:07,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:08,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:11:10,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:12,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:15,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:18,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:11:18,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:21,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:23,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:23,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:11:26,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:28,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:11:28,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:31,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:34,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:36,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:39,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:40,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:11:41,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:44,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:46,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:50,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:50,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:11:52,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:53,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:11:55,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:11:58,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:11:59,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:12:01,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:03,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:06,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:07,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:12:09,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:11,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:14,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:17,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:17,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:19,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:12:19,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:22,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:23,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:12:25,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:27,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:29,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:12:30,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:35,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:39,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:12:40,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:43,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:45,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:49,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:49,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:12:51,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:53,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:12:54,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:12:59,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:12:59,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:02,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:13:03,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:04,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:08,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:09,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:12,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:13:13,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:14,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:18,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:20,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:22,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:13:23,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:13:23,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:25,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:28,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:30,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:33,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:35,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:40,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:41,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:13:41,883 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:13:41,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:13:41,886 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:13:42,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:13:43,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:44,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:13:45,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:48,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:50,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:53,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:13:54,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:13:54,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:13:55,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:13:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:01,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:04,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:14:05,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:06,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:10,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:12,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:14,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:14:16,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:17,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:21,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:23,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:23,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:14:24,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:14:26,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:28,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:31,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:34,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:34,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:14:37,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:39,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:42,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:43,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:14:44,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:47,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:49,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:53,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:14:53,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:14:53,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:14:55,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:14:58,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:00,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:00,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:03,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:15:04,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:05,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:09,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:12,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:14,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:15,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:15:17,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:19,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:23,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:23,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:15:25,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:26,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:15:28,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:30,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:33,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:33,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:35,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:36,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:15:38,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:40,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:43,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:46,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:15:46,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:48,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:51,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:53,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:15:54,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:15:56,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:15:56,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:15:59,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:01,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:04,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:06,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:16:06,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:09,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:11,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:14,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:16,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:20,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:22,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:16:22,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:23,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:16:25,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:27,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:30,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:32,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:16:33,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:35,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:38,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:40,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:42,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:16:44,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:45,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:49,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:51,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:52,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:16:53,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:16:54,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:16:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:16:59,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:02,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:02,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:04,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:04,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:07,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:09,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:12,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:17:12,126 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:17:12,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:17:12,129 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:17:12,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:12,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:17:13,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:14,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:15,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:17,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:20,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:23,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:17:23,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:23,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:25,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:30,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:30,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:30,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:30,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:31,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:35,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:36,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:40,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:42,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:42,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:45,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:47,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:51,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:52,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:17:53,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:17:54,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:17:56,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:17:57,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:02,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:02,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:02,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:06,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:18:07,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:07,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:12,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:13,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:14,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:18:18,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:18,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:23,644 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:18:23,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:23,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:24,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:18:28,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:28,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:33,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:34,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:18:34,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:34,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:38,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:39,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:43,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:44,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:46,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:18:49,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:49,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:53,645 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:18:54,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:18:54,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:18:55,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:18:59,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:01,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:04,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:19:04,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:06,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:09,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:12,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:14,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:19:15,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:17,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:20,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:22,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:23,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:19:24,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:19:25,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:27,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:30,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:33,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:33,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:34,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:19:36,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:38,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:41,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:43,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:44,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:19:47,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:48,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:52,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:19:53,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:19:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:19:55,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:19:57,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:01,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:03,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:05,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:20:06,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:08,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:11,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:12,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:20:13,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:16,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:18,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:22,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:20:23,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:20:23,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:26,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:28,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:33,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:33,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:33,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:33,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:34,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:20:34,703 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:20:34,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:20:34,707 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:20:35,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:20:36,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:20:38,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:38,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:43,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:44,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:46,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:20:49,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:49,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:53,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:20:54,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:20:55,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:20:56,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:20:59,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:00,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:04,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:21:05,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:05,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:10,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:10,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:14,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:21:16,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:16,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:21,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:21,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:23,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:21:26,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:26,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:21:26,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:31,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:31,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:36,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:21:36,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:37,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:37,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:41,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:43,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:46,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:21:47,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:48,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:52,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:53,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:21:53,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:21:54,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:21:57,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:21:58,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:02,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:04,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:04,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:22:08,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:09,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:14,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:14,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:22:15,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:19,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:20,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:23,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:22:24,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:25,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:30,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:30,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:31,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:22:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:35,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:41,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:22:41,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:41,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:46,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:46,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:51,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:22:51,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:51,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:22:53,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:22:56,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:22:57,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:01,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:01,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:03,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:07,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:07,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:08,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:08,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:12,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:13,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:18,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:19,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:20,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:23,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:23,651 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:23:24,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:28,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:28,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:30,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:33,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:35,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:38,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:39,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:41,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:44,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:46,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:46,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:49,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:51,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:53,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:23:54,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:23:54,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:23:54,893 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:23:54,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:23:54,896 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:23:55,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:23:56,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:23:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:23:59,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:01,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:05,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:05,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:06,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:24:07,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:10,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:12,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:15,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:16,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:24:17,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:20,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:22,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:23,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:24:25,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:26,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:24:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:30,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:33,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:34,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:24:36,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:38,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:41,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:44,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:44,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:24:46,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:49,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:52,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:52,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:53,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:24:54,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:24:55,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:24:57,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:24:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:02,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:05,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:25:05,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:07,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:10,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:13,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:15,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:25:15,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:18,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:20,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:23,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:25:23,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:23,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:23,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:25:25,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:28,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:30,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:33,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:25:34,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:35,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:39,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:40,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:45,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:25:45,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:46,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:50,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:50,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:50,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:51,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:25:53,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:25:53,654 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:25:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:25:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:00,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:02,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:03,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:05,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:07,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:11,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:12,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:13,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:16,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:17,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:20,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:21,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:22,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:23,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:26:28,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:28,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:30,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:33,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:34,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:38,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:38,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:39,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:40,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:43,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:44,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:48,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:49,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:50,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:53,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:26:53,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:26:55,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:26:58,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:26:58,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:01,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:03,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:06,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:07,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:27:07,145 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:27:07,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:27:07,149 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:27:07,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:27:08,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:27:09,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:09,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:11,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:15,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:16,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:16,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:27:20,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:21,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:23,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:27:25,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:26,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:29,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:27:30,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:31,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:36,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:36,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:36,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:39,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:27:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:41,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:47,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:47,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:49,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:27:52,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:52,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:27:53,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:27:57,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:27:57,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:27:57,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:02,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:03,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:07,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:28:08,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:08,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:13,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:14,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:17,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:28:18,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:19,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:23,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:28:24,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:24,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:24,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:27,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:28:29,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:29,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:34,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:35,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:37,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:28:40,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:40,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:45,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:45,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:45,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:28:50,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:50,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:28:53,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:28:55,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:28:55,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:55,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:28:55,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:00,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:01,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:05,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:29:06,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:06,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:11,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:12,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:15,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:29:16,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:17,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:21,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:22,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:23,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:29:27,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:28,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:28,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:29:32,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:33,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:37,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:38,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:38,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:29:42,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:44,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:46,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:29:47,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:49,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:52,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:29:53,661 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:29:54,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:54,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:29:57,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:29:58,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:00,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:03,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:05,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:07,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:30:08,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:10,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:13,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:15,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:30:15,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:18,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:20,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:23,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:30:24,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:25,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:30:25,639 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:30:25,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:30:25,642 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:30:26,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:26,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:30:27,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:30:30,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:31,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:35,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:37,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:37,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:30:40,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:43,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:46,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:47,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:30:48,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:51,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:53,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:30:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:30:56,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:30:57,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:30:59,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:01,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:04,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:07,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:07,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:31:09,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:12,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:15,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:17,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:17,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:31:20,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:22,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:23,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:31:25,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:25,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:27,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:27,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:31:30,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:32,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:35,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:37,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:31:37,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:40,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:42,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:46,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:47,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:31:48,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:53,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:31:53,667 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:31:56,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:31:57,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:31:59,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:02,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:04,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:07,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:32:07,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:09,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:13,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:14,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:15,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:32:18,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:19,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:23,667 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:32:23,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:23,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:24,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:25,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:32:28,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:29,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:34,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:35,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:35,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:32:39,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:40,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:44,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:45,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:32:45,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:49,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:50,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:32:53,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:32:55,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:32:55,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:32:56,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:00,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:01,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:05,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:33:06,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:06,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:11,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:11,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:11,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:15,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:33:16,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:16,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:21,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:22,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:23,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:33:23,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:33:27,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:27,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:32,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:35,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:33:37,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:38,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:42,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:33:42,999 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:33:43,000 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:33:43,002 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:33:43,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:43,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:33:44,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:45,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:33:48,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:49,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:53,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:53,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:33:53,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:33:54,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:33:58,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:33:59,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:03,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:03,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:34:05,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:08,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:10,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:13,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:13,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:34:15,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:18,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:23,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:34:24,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:25,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:34:26,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:29,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:31,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:33,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:34:34,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:37,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:40,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:42,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:44,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:34:45,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:47,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:52,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:34:53,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:34:54,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:34:55,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:34:58,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:01,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:02,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:35:04,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:06,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:09,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:11,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:12,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:35:14,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:16,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:19,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:21,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:35:22,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:23,670 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:35:25,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:28,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:30,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:31,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:35:33,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:35,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:38,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:38,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:39,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:35:40,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:43,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:45,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:49,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:49,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:35:51,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:53,671 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:35:54,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:35:57,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:35:59,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:01,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:36:03,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:04,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:08,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:09,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:11,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:36:14,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:14,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:19,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:20,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:21,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:36:23,671 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:36:24,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:25,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:30,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:31,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:31,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:36:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:36,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:41,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:45,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:50,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:51,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:53,672 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:36:56,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:56,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:36:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:36:59,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:01,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:02,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:06,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:07,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:11,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:11,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:12,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:16,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:17,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:19,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:37:19,824 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:37:19,824 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:37:19,827 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:37:20,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:37:21,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:21,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:22,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:22,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:23,672 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:37:27,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:27,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:28,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:32,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:32,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:34,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:38,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:39,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:40,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:43,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:44,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:48,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:49,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:50,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:37:53,673 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:37:53,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:37:55,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:37:59,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:00,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:00,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:04,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:05,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:09,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:10,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:11,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:16,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:16,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:18,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:21,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:22,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:23,675 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:38:26,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:26,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:27,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:28,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:31,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:32,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:36,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:37,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:37,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:41,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:47,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:47,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:47,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:52,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:38:52,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:53,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:38:57,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:38:57,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:38:58,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:02,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:03,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:07,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:39:08,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:08,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:13,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:14,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:14,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:17,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:39:19,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:19,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:23,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:39:24,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:25,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:27,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:39:30,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:30,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:35,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:35,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:37,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:39:40,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:40,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:45,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:46,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:47,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:39:51,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:51,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:53,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:39:56,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:39:57,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:39:58,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:01,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:02,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:07,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:07,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:08,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:12,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:13,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:13,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:18,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:18,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:18,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:23,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:23,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:40:24,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:26,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:29,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:29,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:34,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:34,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:40:35,112 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:40:35,113 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:40:35,115 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:40:35,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:40:35,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:36,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:39,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:40,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:40,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:45,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:45,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:48,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:40:50,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:50,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:53,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:40:55,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:55,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:40:55,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:40:58,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:00,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:00,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:05,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:06,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:10,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:10,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:15,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:16,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:16,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:21,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:22,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:23,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:41:24,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:26,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:27,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:31,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:32,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:34,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:37,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:37,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:42,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:42,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:43,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:47,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:47,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:51,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:41:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:53,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:41:58,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:41:59,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:41:59,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:01,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:42:04,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:04,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:09,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:09,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:11,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:42:14,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:14,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:19,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:19,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:22,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:42:23,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:42:24,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:25,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:30,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:32,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:42:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:35,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:40,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:42:40,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:41,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:46,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:46,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:50,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:42:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:52,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:53,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:42:56,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:42:57,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:42:57,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:00,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:01,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:02,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:06,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:07,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:10,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:12,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:12,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:17,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:18,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:18,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:22,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:23,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:23,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:43:27,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:28,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:28,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:32,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:33,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:37,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:38,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:38,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:42,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:44,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:44,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:43:44,782 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:43:44,783 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:43:44,785 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:43:45,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:43:45,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:46,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:48,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:49,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:53,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:43:54,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:54,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:43:56,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:43:59,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:43:59,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:05,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:05,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:06,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:44:10,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:10,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:15,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:15,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:15,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:44:20,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:22,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:23,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:44:24,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:44:26,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:27,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:31,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:32,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:34,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:44:37,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:37,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:42,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:42,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:42,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:44,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:44:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:48,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:53,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:53,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:44:53,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:44:54,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:44:58,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:44:58,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:03,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:04,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:45:04,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:08,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:09,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:13,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:14,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:45:14,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:19,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:20,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:22,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:45:23,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:45:25,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:25,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:30,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:30,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:30,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:34,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:45:35,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:35,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:40,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:41,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:42,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:45:46,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:46,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:51,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:51,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:45:52,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:45:53,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:45:56,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:45:56,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:01,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:01,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:02,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:46:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:07,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:10,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:46:12,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:12,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:17,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:18,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:20,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:46:22,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:23,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:23,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:46:27,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:28,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:30,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:46:33,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:33,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:38,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:39,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:40,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:46:43,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:44,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:49,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:50,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:51,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:46:53,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:46:54,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:46:56,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:46:58,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:46:58,136 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:46:58,137 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:46:58,140 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:46:59,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:47:00,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:00,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:01,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:01,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:05,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:06,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:08,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:10,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:11,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:15,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:17,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:18,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:21,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:22,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:23,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:47:26,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:27,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:28,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:31,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:32,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:36,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:36,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:37,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:41,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:43,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:43,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:46,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:47,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:48,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:52,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:53,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:47:54,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:47:56,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:47:58,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:47:59,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:03,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:04,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:06,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:48:08,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:09,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:13,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:15,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:16,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:48:19,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:20,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:23,690 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:48:24,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:48:24,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:25,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:29,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:31,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:34,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:48:35,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:36,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:40,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:42,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:44,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:48:45,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:47,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:50,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:52,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:48:52,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:48:53,691 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:48:56,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:48:57,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:01,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:03,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:04,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:49:07,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:08,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:12,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:12,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:49:14,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:17,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:19,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:23,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:49:23,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:23,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:49:24,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:28,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:30,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:30,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:33,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:49:34,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:39,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:40,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:41,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:49:44,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:45,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:50,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:51,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:49:51,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:49:53,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:49:55,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:49:56,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:01,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:01,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:50:01,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:06,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:06,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:08,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:50:08,409 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:50:08,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:50:08,412 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:50:09,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:50:10,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:50:11,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:12,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:17,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:17,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:17,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:18,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:50:22,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:22,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:23,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:50:27,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:28,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:50:32,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:33,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:37,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:38,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:50:39,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:42,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:44,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:48,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:48,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:50:50,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:53,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:53,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:50:55,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:50:58,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:50:58,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:51:01,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:03,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:06,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:09,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:11,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:12,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:51:14,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:16,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:20,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:21,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:22,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:51:23,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:51:25,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:26,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:30,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:30,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:51:32,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:35,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:37,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:40,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:40,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:51:42,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:45,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:47,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:51,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:51:51,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:52,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:53,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:51:56,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:51:57,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:51:59,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:01,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:02,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:06,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:09,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:12,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:12,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:17,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:17,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:18,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:22,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:23,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:23,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:52:27,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:27,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:28,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:32,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:33,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:37,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:37,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:39,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:44,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:47,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:48,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:49,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:53,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:52:53,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:52:55,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:52:55,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:52:58,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:00,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:04,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:05,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:05,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:09,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:10,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:12,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:15,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:15,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:20,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:20,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:21,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:53:21,919 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:53:21,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:53:21,923 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:53:22,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:53:22,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:23,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:53:24,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:25,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:25,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:30,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:31,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:32,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:36,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:36,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:41,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:41,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:42,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:46,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:46,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:50,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:53:51,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:53:51,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:53,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:53:57,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:53:57,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:00,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:54:02,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:02,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:07,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:08,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:08,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:54:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:13,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:18,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:19,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:54:19,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:23,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:54:24,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:25,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:27,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:54:30,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:30,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:35,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:35,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:35,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:37,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:54:40,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:40,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:45,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:45,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:47,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:54:50,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:50,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:53,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:54:55,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:54:55,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:54:57,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:00,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:00,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:05,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:06,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:07,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:10,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:11,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:15,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:16,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:17,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:22,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:22,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:23,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:55:25,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:27,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:27,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:32,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:33,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:35,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:38,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:38,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:43,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:44,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:44,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:49,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:50,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:53,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:55:53,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:55:54,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:55:55,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:55:59,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:00,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:01,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:05,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:05,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:10,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:10,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:11,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:15,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:15,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:20,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:20,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:20,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:23,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:56:26,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:27,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:28,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:56:28,624 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:56:28,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:56:28,627 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:56:28,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:56:28,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:30,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:32,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:32,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:37,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:37,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:37,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:38,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:42,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:42,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:48,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:49,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:52,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:53,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:56:53,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:56:57,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:56:58,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:56:58,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:03,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:04,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:04,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:07,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:57:08,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:09,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:13,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:14,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:17,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:57:18,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:19,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:23,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:57:24,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:25,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:25,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:57:29,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:30,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:34,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:35,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:57:35,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:39,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:43,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:57:44,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:46,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:49,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:53,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:57:53,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:57:54,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:57:56,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:57:59,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:01,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:58:01,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:05,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:06,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:10,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:11,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:58:12,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:15,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:18,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:18,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:21,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:21,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:58:23,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:58:23,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:26,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:29,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:31,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:58:32,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:34,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:37,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:39,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:39,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:58:42,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:44,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:47,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:49,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:49,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:58:52,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:53,705 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:58:55,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:58:57,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:58:59,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:00,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:02,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:07,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:11,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:13,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:16,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:17,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:18,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:21,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:23,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:23,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:59:25,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:26,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:28,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:32,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:33,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:34,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 00:59:34,041 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 00:59:34,042 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 00:59:34,044 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 00:59:34,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 00:59:35,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:37,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:39,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:43,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:44,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:45,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:48,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:49,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:53,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:53,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:53,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 00:59:54,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 00:59:55,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 00:59:58,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 00:59:59,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:03,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:00:03,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:03,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:04,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:08,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:09,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:13,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:00:14,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:15,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:20,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:20,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:20,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:23,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:00:23,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:00:25,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:25,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:30,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:31,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:31,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:00:35,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:36,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:40,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:41,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:00:41,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:45,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:46,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:51,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:51,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:00:52,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:00:53,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:00:56,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:00:57,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:01,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:01,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:01:03,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:07,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:07,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:08,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:09,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:01:12,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:13,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:17,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:18,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:19,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:01:22,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:23,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:01:24,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:27,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:01:28,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:30,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:35,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:35,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:37,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:01:40,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:40,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:45,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:45,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:48,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:01:50,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:50,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:53,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:01:55,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:01:55,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:01:56,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:00,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:00,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:05,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:06,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:06,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:11,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:11,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:14,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:16,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:17,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:22,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:22,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:22,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:23,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:23,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:02:27,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:27,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:32,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:32,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:33,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:37,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:37,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:40,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:02:40,922 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:02:40,923 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:02:40,926 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:02:41,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:02:41,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:42,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:43,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:43,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:47,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:48,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:51,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:02:52,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:53,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:02:53,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:02:58,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:02:59,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:01,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:03:04,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:04,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:09,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:03:09,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:09,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:14,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:14,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:19,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:03:20,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:20,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:23,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:03:25,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:26,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:29,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:03:31,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:31,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:36,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:36,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:37,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:03:41,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:42,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:47,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:47,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:47,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:03:52,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:52,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:53,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:03:57,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:03:57,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:03:57,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:02,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:03,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:05,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:08,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:08,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:13,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:14,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:16,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:19,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:19,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:23,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:04:24,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:24,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:26,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:29,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:29,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:34,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:34,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:35,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:40,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:40,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:40,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:44,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:45,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:46,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:04:51,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:53,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:04:54,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:04:56,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:04:56,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:01,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:01,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:06,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:05:06,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:06,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:11,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:13,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:16,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:05:17,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:18,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:22,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:23,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:23,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:23,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:05:25,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:05:27,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:28,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:32,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:33,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:05:34,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:38,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:39,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:43,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:43,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:05:44,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:48,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:50,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:51,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:05:51,753 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:05:51,754 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:05:51,756 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:05:52,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:05:53,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:05:53,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:05:53,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:05:55,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:05:58,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:01,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:01,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:04,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:06,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:09,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:11,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:11,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:11,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:14,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:16,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:19,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:21,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:21,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:23,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:06:24,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:27,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:29,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:29,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:32,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:34,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:37,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:39,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:40,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:42,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:45,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:48,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:50,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:50,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:53,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:06:53,716 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:06:55,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:06:58,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:06:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:00,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:03,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:05,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:08,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:07:08,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:11,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:14,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:16,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:18,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:07:19,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:21,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:23,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:07:24,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:24,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:26,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:07:26,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:29,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:31,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:34,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:36,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:07:37,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:39,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:44,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:07:44,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:48,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:49,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:53,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:07:53,717 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:07:54,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:07:58,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:00,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:04,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:04,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:08:05,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:09,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:11,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:11,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:14,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:08:14,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:19,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:21,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:22,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:08:23,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:08:25,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:26,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:31,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:31,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:32,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:08:36,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:36,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:41,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:41,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:08:42,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:46,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:47,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:49,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:08:51,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:52,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:53,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:08:57,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:08:58,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:08:59,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:03,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:03,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:07,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:09:07,859 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:09:07,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:09:07,900 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:09:07,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:09:08,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:08,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:08,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:09,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:13,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:13,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:17,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:19,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:19,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:23,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:09:24,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:25,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:25,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:28,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:30,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:30,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:35,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:35,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:36,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:40,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:40,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:46,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:46,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:46,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:51,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:52,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:09:53,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:09:56,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:09:56,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:09:57,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:01,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:03,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:06,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:10:07,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:12,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:14,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:14,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:14,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:10:18,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:19,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:23,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:10:23,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:24,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:10:24,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:28,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:30,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:32,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:10:33,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:35,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:38,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:40,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:42,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:10:43,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:45,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:48,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:50,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:10:51,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:53,722 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:10:54,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:10:56,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:10:59,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:00,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:01,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:04,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:07,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:10,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:10,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:12,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:15,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:17,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:18,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:20,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:22,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:23,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:11:25,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:28,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:28,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:30,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:33,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:35,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:38,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:38,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:43,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:47,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:47,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:49,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:52,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:53,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:11:54,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:11:58,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:11:58,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:11:59,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:03,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:05,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:08,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:08,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:10,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:13,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:15,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:12:15,392 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:12:15,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:12:15,395 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:12:16,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:16,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:12:16,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:18,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:19,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:21,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:23,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:12:24,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:26,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:26,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:29,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:31,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:34,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:37,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:38,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:40,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:42,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:45,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:48,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:48,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:51,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:53,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:12:54,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:12:57,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:12:58,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:12:59,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:02,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:05,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:06,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:13:07,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:10,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:12,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:15,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:16,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:13:17,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:20,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:22,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:23,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:13:24,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:13:25,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:27,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:30,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:32,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:34,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:13:35,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:37,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:41,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:41,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:43,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:44,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:13:46,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:48,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:52,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:52,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:13:53,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:13:54,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:13:57,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:13:59,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:02,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:02,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:14:05,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:07,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:10,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:12,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:15,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:17,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:20,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:22,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:23,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:14:25,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:27,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:30,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:32,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:36,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:37,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:14:37,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:41,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:43,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:43,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:47,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:47,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:14:48,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:52,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:53,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:14:54,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:14:54,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:14:57,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:14:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:02,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:04,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:04,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:08,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:09,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:14,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:14,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:15,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:15,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:19,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:20,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:23,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:15:24,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:24,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:25,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:29,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:30,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:30,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:32,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:35,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:35,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:40,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:41,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:42,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:45,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:46,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:49,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:15:49,588 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:15:49,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:15:49,590 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:15:49,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:15:50,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:50,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:51,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:15:51,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:52,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:15:53,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:15:55,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:15:56,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:00,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:01,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:05,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:06,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:10,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:11,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:15,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:16,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:20,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:21,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:23,730 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:16:26,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:26,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:31,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:36,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:36,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:41,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:41,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:46,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:46,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:51,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:51,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:16:53,732 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:16:56,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:16:57,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:01,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:06,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:07,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:11,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:12,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:16,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:17,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:21,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:22,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:23,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:17:26,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:28,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:31,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:33,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:36,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:38,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:41,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:43,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:46,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:48,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:51,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:53,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:17:54,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:17:56,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:17:59,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:01,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:04,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:06,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:09,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:11,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:14,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:16,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:19,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:21,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:23,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:18:25,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:26,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:30,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:31,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:35,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:36,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:40,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:41,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:45,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:46,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:18:51,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:53,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:18:56,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:18:56,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:01,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:01,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:06,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:06,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:11,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:11,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:16,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:16,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:21,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:21,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:23,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:19:26,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:27,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:31,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:32,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:36,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:37,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:41,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:42,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:46,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:47,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:51,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:19:53,735 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:19:56,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:19:58,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:01,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:03,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:06,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:08,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:11,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:13,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:16,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:18,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:21,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:23,735 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:20:24,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:26,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:29,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:31,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:34,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:36,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:39,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:41,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:44,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:46,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:49,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:51,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:20:53,735 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:20:55,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:20:56,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:00,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:01,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:05,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:06,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:10,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:11,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:15,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:16,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:20,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:21,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:23,736 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:21:26,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:26,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:31,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:31,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:36,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:36,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:41,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:41,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:46,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:46,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:51,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:51,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:21:53,736 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:21:56,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:21:57,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:01,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:02,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:07,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:11,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:12,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:16,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:17,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:21,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:22,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:23,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:22:26,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:28,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:31,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:33,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:36,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:38,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:41,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:43,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:46,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:48,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:51,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:53,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:22:54,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:22:56,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:22:59,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:01,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:04,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:06,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:09,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:11,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:14,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:16,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:19,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:21,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:23,738 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:23:25,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:26,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:30,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:31,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:35,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:36,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:40,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:41,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:45,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:46,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:50,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:51,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:23:53,738 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:23:55,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:23:56,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:00,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:01,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:05,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:10,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:11,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:15,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:16,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:20,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:21,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:23,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:24:26,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:26,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:31,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:31,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:36,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:36,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:41,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:41,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:46,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:46,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:51,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:51,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:24:53,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:24:56,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:24:57,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:01,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:02,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:06,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:07,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:11,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:12,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:16,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:17,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:21,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:22,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:23,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:25:26,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:28,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:31,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:33,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:36,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:38,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:41,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:25:41,411 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:25:41,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:25:41,415 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:25:41,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:42,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:25:42,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:25:43,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:25:44,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:46,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:49,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:51,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:53,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:25:54,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:25:56,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:25:59,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:01,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:04,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:06,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:09,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:11,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:13,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:26:15,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:17,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:20,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:22,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:23,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:26:25,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:27,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:30,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:33,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:35,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:38,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:40,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:44,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:45,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:50,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:51,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:52,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:26:53,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:26:53,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:26:56,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:26:56,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:56,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:56,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:26:58,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:27:01,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:03,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:06,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:11,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:16,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:16,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:17,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:18,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:27:21,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:22,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:23,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:27:26,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:27,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:31,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:32,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:36,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:37,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:41,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:42,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:46,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:47,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:51,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:52,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:27:53,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:27:56,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:27:58,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:01,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:03,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:06,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:08,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:11,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:13,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:16,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:28:16,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:19,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:21,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:23,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:28:24,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:26,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:27,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:28:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:31,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:34,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:36,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:37,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:28:40,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:43,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:45,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:28:45,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:48,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:50,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:53,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:28:53,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:28:55,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:28:56,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:28:58,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:01,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:04,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:04,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:05,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:29:06,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:09,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:11,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:15,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:15,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:29:16,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:20,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:21,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:23,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:29:25,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:29:25,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:27,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:30,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:32,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:35,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:29:36,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:36,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:37,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:41,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:42,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:45,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:29:46,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:47,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:52,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:53,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:29:53,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:29:53,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:29:57,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:29:58,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:02,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:02,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:03,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:30:04,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:07,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:09,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:13,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:13,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:30:15,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:19,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:19,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:20,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:23,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:30:23,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:30:24,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:25,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:30,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:31,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:32,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:30:35,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:36,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:40,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:41,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:42,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:30:45,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:46,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:51,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:51,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:52,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:30:52,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:30:53,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:30:56,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:30:57,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:01,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:02,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:31:02,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:06,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:07,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:11,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:12,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:31:13,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:17,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:18,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:20,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:31:22,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:23,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:23,746 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:31:28,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:31:28,038 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:31:28,038 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:31:28,041 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:31:28,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:28,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:31:29,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:29,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:31:33,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:34,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:38,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:39,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:39,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:31:43,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:44,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:47,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:31:48,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:49,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:53,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:31:53,747 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:31:55,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:31:58,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:00,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:03,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:32:04,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:05,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:09,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:10,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:13,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:32:14,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:16,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:20,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:20,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:21,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:23,747 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:32:23,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:32:25,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:26,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:30,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:31,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:35,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:36,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:40,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:41,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:45,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:46,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:50,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:51,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:53,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:32:56,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:32:57,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:32:59,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:33:01,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:02,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:06,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:08,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:08,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:33:11,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:13,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:17,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:18,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:20,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:33:22,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:23,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:33:24,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:28,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:30,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:33:30,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:33,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:35,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:38,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:40,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:33:40,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:43,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:45,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:49,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:50,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:33:51,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:53,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:33:55,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:33:56,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:33:58,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:00,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:01,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:06,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:06,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:06,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:08,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:11,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:16,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:17,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:18,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:21,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:22,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:23,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:34:26,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:27,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:28,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:31,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:32,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:37,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:37,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:38,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:42,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:43,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:45,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:47,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:49,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:53,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:53,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:53,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:34:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:34:55,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:34:58,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:34:59,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:03,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:04,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:05,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:08,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:09,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:12,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:35:12,928 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:35:12,929 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:35:12,931 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:35:13,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:13,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:35:13,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:14,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:14,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:18,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:19,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:23,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:35:23,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:24,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:25,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:30,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:30,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:34,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:35,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:36,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:41,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:41,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:44,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:46,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:46,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:51,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:51,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:35:52,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:35:53,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:35:56,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:35:57,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:02,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:36:02,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:03,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:08,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:08,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:08,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:12,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:36:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:13,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:18,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:19,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:22,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:36:23,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:36:24,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:24,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:30,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:36:34,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:35,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:40,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:40,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:36:40,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:45,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:46,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:50,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:36:51,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:51,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:36:53,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:36:56,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:36:57,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:00,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:37:02,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:02,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:07,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:10,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:37:12,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:13,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:18,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:18,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:19,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:37:23,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:23,756 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:37:24,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:29,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:29,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:37:30,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:35,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:35,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:35,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:38,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:37:40,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:41,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:46,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:46,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:48,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:37:51,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:51,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:53,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:37:57,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:37:57,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:37:58,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:02,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:03,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:06,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:07,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:08,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:12,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:13,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:16,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:18,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:18,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:23,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:23,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-21 01:38:23,606 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-21 01:38:23,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-21 01:38:23,609 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-21 01:38:23,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:38:24,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-21 01:38:24,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:24,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:25,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:26,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:28,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:29,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:34,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:34,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:35,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:39,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:41,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:44,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:44,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:46,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:49,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:51,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:53,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:38:54,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:38:54,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:38:56,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:38:59,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:02,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:39:02,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:39:05,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:07,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:39:10,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:12,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:39:12,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:39:15,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:17,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:39:20,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:22,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:39:22,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:39:23,758 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-21 01:39:26,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:27,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-21 01:39:32,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-21 01:39:32,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-21 01:39:33,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive